From ae266ed50e7a22bc320bfb98a92dc874980a3bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieliski?= Date: Sun, 25 Aug 2024 16:45:36 +0200 Subject: [PATCH] QueueJob - model and DB migration --- WebAPI/AppDbContext.cs | 9 +- WebAPI/Controllers/LayersController.cs | 89 +++- WebAPI/Controllers/LogsController.cs | 1 + WebAPI/DiunaBI-WebAPI.csproj | 5 + .../20240825144443_QueueJobs.Designer.cs | 381 ++++++++++++++++++ WebAPI/Migrations/20240825144443_QueueJobs.cs | 40 ++ .../Migrations/AppDbContextModelSnapshot.cs | 33 ++ WebAPI/Models/LogEntry.cs | 3 +- WebAPI/Models/QueueJob.cs | 29 ++ 9 files changed, 578 insertions(+), 12 deletions(-) create mode 100644 WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs create mode 100644 WebAPI/Migrations/20240825144443_QueueJobs.cs create mode 100644 WebAPI/Models/QueueJob.cs diff --git a/WebAPI/AppDbContext.cs b/WebAPI/AppDbContext.cs index 0ba188f..f0e910b 100644 --- a/WebAPI/AppDbContext.cs +++ b/WebAPI/AppDbContext.cs @@ -3,17 +3,14 @@ using WebAPI.Models; namespace WebAPI; -public class AppDbContext : DbContext +public class AppDbContext(DbContextOptions options) : DbContext(options) { public DbSet Users { get; init; } public DbSet Layers { get; init; } public DbSet Records { get; init; } public DbSet ProcessSources { get; init; } - public DbSet DataInbox { get; set; } - - public AppDbContext(DbContextOptions options) : base(options) - { - } + public DbSet DataInbox { get; init; } + public DbSet QueueJobs { get; init; } protected override void OnModelCreating(ModelBuilder modelBuilder) { diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index 82d9d6f..36361a4 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -26,7 +26,7 @@ public class LayersController : Controller GoogleSheetsHelper googleSheetsHelper, GoogleDriveHelper googleDriveHelper, IConfiguration configuration - ) + ) { _db = db; if (googleSheetsHelper.Service is not null) @@ -182,6 +182,83 @@ public class LayersController : Controller return Ok(true); } + [HttpGet] + [Route("AutoImportWithQueue/{apiKey}")] + [AllowAnonymous] + public async Task AutoImportWithQueue(string apiKey) + { + if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) + { + return Unauthorized(); + } + + var importWorkerLayers = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + ) + .OrderBy(x => x.CreatedAt) + .ToList(); + + if (importWorkerLayers.Count == 0) + { + _logsController.AddEntry(new LogEntry + { + Title = "No Layers to import.", + Type = LogEntryType.Info, + LogType = LogType.Queue, + CreatedAt = DateTime.UtcNow + }); + return Ok(); + } + + foreach (var importWorker in importWorkerLayers) + { + try + { + /* + await _queue.AddJob(new QueueJob + { + LayerId = importWorker.Id, + Type = JobType.ImportWorker, + }); + */ + } + catch (Exception e) + { + _logsController.AddEntry(new LogEntry + { + Title = $"Error while adding job into queue (import): {importWorker.Name}, {importWorker.Id}", + Type = LogEntryType.Error, + LogType = LogType.Queue, + Message = e.ToString(), + CreatedAt = DateTime.UtcNow + }); + } + } + return Ok(); + } + + [HttpGet] + [Route("ProcessQueue/{apiKey}")] + [AllowAnonymous] + public async Task ProcessQueue(string apiKey) + { + /* + var allJobs = await _queue.GetJobs(); + var importJobs = allJobs + .Where(x => x.Type == JobType.ImportWorker && x.Status == JobStatus.New); + + foreach (var job in importJobs) + { + job.Attempts = job.Attempts + 1; + //await _queue.UpdateJob(job); + } + */ + return Ok(); + } + [HttpGet] [Route("AutoImport/{apiKey}")] [AllowAnonymous] @@ -200,8 +277,8 @@ public class LayersController : Controller var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") ) .OrderBy(x => x.CreatedAt) .ToList(); @@ -231,7 +308,7 @@ public class LayersController : Controller var importer = new MorskaFk2Importer(_db, _googleSheetValues, this); importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - + _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", @@ -665,9 +742,10 @@ public class LayersController : Controller if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) || - double.Abs((double)(record.Value1-value)!) < 0.01) continue; + double.Abs((double)(record.Value1 - value)!) < 0.01) continue; isUpToDate = false; } + foreach (var record in newestLayer.Records!) { if (data[0].Contains(record.Code)) @@ -678,6 +756,7 @@ public class LayersController : Controller WriteToConsole($"Code not found in GoogleSheet: {record.Code}"); isUpToDate = false; } + return isUpToDate; } } \ No newline at end of file diff --git a/WebAPI/Controllers/LogsController.cs b/WebAPI/Controllers/LogsController.cs index 08dc45d..a011b40 100644 --- a/WebAPI/Controllers/LogsController.cs +++ b/WebAPI/Controllers/LogsController.cs @@ -33,6 +33,7 @@ public class LogsController : Controller LogType.Process => "Process", LogType.PowerBi => "PowerBIAccess", LogType.DataInbox => "DataInbox", + LogType.Queue => "Queue", _ => "Other" }; var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute(); diff --git a/WebAPI/DiunaBI-WebAPI.csproj b/WebAPI/DiunaBI-WebAPI.csproj index 9c6636b..151f4c1 100644 --- a/WebAPI/DiunaBI-WebAPI.csproj +++ b/WebAPI/DiunaBI-WebAPI.csproj @@ -20,6 +20,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + @@ -36,4 +37,8 @@ + + + + diff --git a/WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs b/WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs new file mode 100644 index 0000000..e1d0bd3 --- /dev/null +++ b/WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs @@ -0,0 +1,381 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using WebAPI; + +#nullable disable + +namespace WebAPI.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20240825144443_QueueJobs")] + partial class QueueJobs + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.6") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("WebAPI.Models.DataInbox", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Data") + .IsRequired() + .HasMaxLength(2147483647) + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("DataInbox"); + }); + + modelBuilder.Entity("WebAPI.Models.Layer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Number") + .HasColumnType("int"); + + b.Property("ParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ModifiedById"); + + b.HasIndex("ParentId"); + + b.ToTable("Layers"); + }); + + modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + { + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("LayerId", "SourceId"); + + b.HasIndex("SourceId"); + + b.ToTable("ProcessSources"); + }); + + modelBuilder.Entity("WebAPI.Models.QueueJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Attempts") + .HasColumnType("int"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.ToTable("QueueJobs"); + }); + + modelBuilder.Entity("WebAPI.Models.Record", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Code") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Desc1") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Value1") + .HasColumnType("float"); + + b.Property("Value10") + .HasColumnType("float"); + + b.Property("Value11") + .HasColumnType("float"); + + b.Property("Value12") + .HasColumnType("float"); + + b.Property("Value13") + .HasColumnType("float"); + + b.Property("Value14") + .HasColumnType("float"); + + b.Property("Value15") + .HasColumnType("float"); + + b.Property("Value16") + .HasColumnType("float"); + + b.Property("Value17") + .HasColumnType("float"); + + b.Property("Value18") + .HasColumnType("float"); + + b.Property("Value19") + .HasColumnType("float"); + + b.Property("Value2") + .HasColumnType("float"); + + b.Property("Value20") + .HasColumnType("float"); + + b.Property("Value21") + .HasColumnType("float"); + + b.Property("Value22") + .HasColumnType("float"); + + b.Property("Value23") + .HasColumnType("float"); + + b.Property("Value24") + .HasColumnType("float"); + + b.Property("Value25") + .HasColumnType("float"); + + b.Property("Value26") + .HasColumnType("float"); + + b.Property("Value27") + .HasColumnType("float"); + + b.Property("Value28") + .HasColumnType("float"); + + b.Property("Value29") + .HasColumnType("float"); + + b.Property("Value3") + .HasColumnType("float"); + + b.Property("Value30") + .HasColumnType("float"); + + b.Property("Value31") + .HasColumnType("float"); + + b.Property("Value32") + .HasColumnType("float"); + + b.Property("Value4") + .HasColumnType("float"); + + b.Property("Value5") + .HasColumnType("float"); + + b.Property("Value6") + .HasColumnType("float"); + + b.Property("Value7") + .HasColumnType("float"); + + b.Property("Value8") + .HasColumnType("float"); + + b.Property("Value9") + .HasColumnType("float"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("LayerId"); + + b.HasIndex("ModifiedById"); + + b.ToTable("Records"); + }); + + modelBuilder.Entity("WebAPI.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Email") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("UserName") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("WebAPI.Models.Layer", b => + { + b.HasOne("WebAPI.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("WebAPI.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("WebAPI.Models.Layer", "Parent") + .WithMany() + .HasForeignKey("ParentId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + + b.Navigation("Parent"); + }); + + modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + { + b.HasOne("WebAPI.Models.Layer", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("WebAPI.Models.Record", b => + { + b.HasOne("WebAPI.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("WebAPI.Models.Layer", null) + .WithMany("Records") + .HasForeignKey("LayerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("WebAPI.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + }); + + modelBuilder.Entity("WebAPI.Models.Layer", b => + { + b.Navigation("Records"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/WebAPI/Migrations/20240825144443_QueueJobs.cs b/WebAPI/Migrations/20240825144443_QueueJobs.cs new file mode 100644 index 0000000..267a4e4 --- /dev/null +++ b/WebAPI/Migrations/20240825144443_QueueJobs.cs @@ -0,0 +1,40 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace WebAPI.Migrations +{ + /// + public partial class QueueJobs : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "QueueJobs", + columns: table => new + { + Id = table.Column(type: "uniqueidentifier", nullable: false), + LayerId = table.Column(type: "uniqueidentifier", nullable: false), + Attempts = table.Column(type: "int", nullable: false), + Status = table.Column(type: "int", nullable: false), + Type = table.Column(type: "int", nullable: false), + Message = table.Column(type: "nvarchar(max)", nullable: false), + CreatedAt = table.Column(type: "datetime2", nullable: false), + ModifiedAt = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_QueueJobs", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "QueueJobs"); + } + } +} diff --git a/WebAPI/Migrations/AppDbContextModelSnapshot.cs b/WebAPI/Migrations/AppDbContextModelSnapshot.cs index 61b7c8a..606c1dd 100644 --- a/WebAPI/Migrations/AppDbContextModelSnapshot.cs +++ b/WebAPI/Migrations/AppDbContextModelSnapshot.cs @@ -112,6 +112,39 @@ namespace WebAPI.Migrations b.ToTable("ProcessSources"); }); + modelBuilder.Entity("WebAPI.Models.QueueJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Attempts") + .HasColumnType("int"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.ToTable("QueueJobs"); + }); + modelBuilder.Entity("WebAPI.Models.Record", b => { b.Property("Id") diff --git a/WebAPI/Models/LogEntry.cs b/WebAPI/Models/LogEntry.cs index ad4a9ab..4dccaf0 100644 --- a/WebAPI/Models/LogEntry.cs +++ b/WebAPI/Models/LogEntry.cs @@ -11,7 +11,8 @@ public enum LogType { Backup, Process, PowerBi, - DataInbox + DataInbox, + Queue } public class LogEntry { diff --git a/WebAPI/Models/QueueJob.cs b/WebAPI/Models/QueueJob.cs new file mode 100644 index 0000000..c122827 --- /dev/null +++ b/WebAPI/Models/QueueJob.cs @@ -0,0 +1,29 @@ +using System.ComponentModel.DataAnnotations; +using System.Security.Cryptography.X509Certificates; + +namespace WebAPI.Models; + +public enum JobStatus +{ + New, + Failed, + Success +} + +public enum JobType +{ + ImportWorker, + ProcessWorker +} + +public class QueueJob +{ + [Key] public Guid Id { get; set; } + [Required] public Guid LayerId { get; set; } + [Required] public int Attempts { get; set; } + [Required] public JobStatus Status { get; set; } = JobStatus.New; + [Required] public JobType Type { get; set; } = JobType.ImportWorker; + public string Message { get; set; } = string.Empty; + [Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + [Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow; +} \ No newline at end of file