mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QueueJob - model and DB migration

Browse Source
This commit is contained in:
Michał Zieliski
2024-08-25 16:45:36 +02:00
parent adc3ba0c4b
commit ae266ed50e
9 changed files with 578 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
WebAPI/AppDbContext.cs
View File

@@ -3,17 +3,14 @@ using WebAPI.Models;
namespace WebAPI;
public class AppDbContext : DbContext
public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
{
public DbSet<User> Users { get; init; }
public DbSet<Layer> Layers { get; init; }
public DbSet<Record> Records { get; init; }
public DbSet<ProcessSource> ProcessSources { get; init; }
public DbSet<DataInbox> DataInbox { get; set; }
public AppDbContext(DbContextOptions<AppDbContext> options) : base(options)
{
}
public DbSet<DataInbox> DataInbox { get; init; }
public DbSet<QueueJob> QueueJobs { get; init; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{

89
WebAPI/Controllers/LayersController.cs
View File

@@ -26,7 +26,7 @@ public class LayersController : Controller
GoogleSheetsHelper googleSheetsHelper,
GoogleDriveHelper googleDriveHelper,
IConfiguration configuration
)
)
{
_db = db;
if (googleSheetsHelper.Service is not null)
@@ -182,6 +182,83 @@ public class LayersController : Controller
return Ok(true);
}
[HttpGet]
[Route("AutoImportWithQueue/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> AutoImportWithQueue(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
return Unauthorized();
}
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
)
.OrderBy(x => x.CreatedAt)
.ToList();
if (importWorkerLayers.Count == 0)
{
_logsController.AddEntry(new LogEntry
{
Title = "No Layers to import.",
Type = LogEntryType.Info,
LogType = LogType.Queue,
CreatedAt = DateTime.UtcNow
});
return Ok();
}
foreach (var importWorker in importWorkerLayers)
{
try
{
/*
await _queue.AddJob(new QueueJob
{
LayerId = importWorker.Id,
Type = JobType.ImportWorker,
});
*/
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = $"Error while adding job into queue (import): {importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Error,
LogType = LogType.Queue,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
}
}
return Ok();
}
[HttpGet]
[Route("ProcessQueue/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> ProcessQueue(string apiKey)
{
/*
var allJobs = await _queue.GetJobs();
var importJobs = allJobs
.Where(x => x.Type == JobType.ImportWorker && x.Status == JobStatus.New);
foreach (var job in importJobs)
{
job.Attempts = job.Attempts + 1;
//await _queue.UpdateJob(job);
}
*/
return Ok();
}
[HttpGet]
[Route("AutoImport/{apiKey}")]
[AllowAnonymous]
@@ -200,8 +277,8 @@ public class LayersController : Controller
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
)
.OrderBy(x => x.CreatedAt)
.ToList();
@@ -231,7 +308,7 @@ public class LayersController : Controller
var importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
@@ -665,9 +742,10 @@ public class LayersController : Controller
if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"),
out var value) ||
double.Abs((double)(record.Value1-value)!) < 0.01) continue;
double.Abs((double)(record.Value1 - value)!) < 0.01) continue;
isUpToDate = false;
}
foreach (var record in newestLayer.Records!)
{
if (data[0].Contains(record.Code))
@@ -678,6 +756,7 @@ public class LayersController : Controller
WriteToConsole($"Code not found in GoogleSheet: {record.Code}");
isUpToDate = false;
}
return isUpToDate;
}
}

1
WebAPI/Controllers/LogsController.cs
View File

@@ -33,6 +33,7 @@ public class LogsController : Controller
LogType.Process => "Process",
LogType.PowerBi => "PowerBIAccess",
LogType.DataInbox => "DataInbox",
LogType.Queue => "Queue",
_ => "Other"
};
var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();

5
WebAPI/DiunaBI-WebAPI.csproj
View File

@@ -20,6 +20,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="StackExchange.Redis" Version="2.8.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
</ItemGroup>
@@ -36,4 +37,8 @@
<None Remove="dataProcessors\" />
<None Remove="dataImporters\" />
</ItemGroup>
<ItemGroup>
<Compile Remove="Controllers\QueueController.cs" />
</ItemGroup>
</Project>

381
WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs generated Normal file
View File

@@ -0,0 +1,381 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using WebAPI;
#nullable disable
namespace WebAPI.Migrations
{
[DbContext(typeof(AppDbContext))]
[Migration("20240825144443_QueueJobs")]
partial class QueueJobs
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "8.0.6")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
modelBuilder.Entity("WebAPI.Models.DataInbox", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<string>("Data")
.IsRequired()
.HasMaxLength(2147483647)
.HasColumnType("nvarchar(max)");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<string>("Source")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.HasKey("Id");
b.ToTable("DataInbox");
});
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("CreatedById")
.HasColumnType("uniqueidentifier");
b.Property<bool>("IsDeleted")
.HasColumnType("bit");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<Guid>("ModifiedById")
.HasColumnType("uniqueidentifier");
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<int>("Number")
.HasColumnType("int");
b.Property<Guid?>("ParentId")
.HasColumnType("uniqueidentifier");
b.Property<int>("Type")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("CreatedById");
b.HasIndex("ModifiedById");
b.HasIndex("ParentId");
b.ToTable("Layers");
});
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
{
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<Guid>("SourceId")
.HasColumnType("uniqueidentifier");
b.HasKey("LayerId", "SourceId");
b.HasIndex("SourceId");
b.ToTable("ProcessSources");
});
modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<int>("Attempts")
.HasColumnType("int");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<string>("Message")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<int>("Status")
.HasColumnType("int");
b.Property<int>("Type")
.HasColumnType("int");
b.HasKey("Id");
b.ToTable("QueueJobs");
});
modelBuilder.Entity("WebAPI.Models.Record", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<string>("Code")
.IsRequired()
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("CreatedById")
.HasColumnType("uniqueidentifier");
b.Property<string>("Desc1")
.HasMaxLength(1000)
.HasColumnType("nvarchar(1000)");
b.Property<bool>("IsDeleted")
.HasColumnType("bit");
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<Guid>("ModifiedById")
.HasColumnType("uniqueidentifier");
b.Property<double?>("Value1")
.HasColumnType("float");
b.Property<double?>("Value10")
.HasColumnType("float");
b.Property<double?>("Value11")
.HasColumnType("float");
b.Property<double?>("Value12")
.HasColumnType("float");
b.Property<double?>("Value13")
.HasColumnType("float");
b.Property<double?>("Value14")
.HasColumnType("float");
b.Property<double?>("Value15")
.HasColumnType("float");
b.Property<double?>("Value16")
.HasColumnType("float");
b.Property<double?>("Value17")
.HasColumnType("float");
b.Property<double?>("Value18")
.HasColumnType("float");
b.Property<double?>("Value19")
.HasColumnType("float");
b.Property<double?>("Value2")
.HasColumnType("float");
b.Property<double?>("Value20")
.HasColumnType("float");
b.Property<double?>("Value21")
.HasColumnType("float");
b.Property<double?>("Value22")
.HasColumnType("float");
b.Property<double?>("Value23")
.HasColumnType("float");
b.Property<double?>("Value24")
.HasColumnType("float");
b.Property<double?>("Value25")
.HasColumnType("float");
b.Property<double?>("Value26")
.HasColumnType("float");
b.Property<double?>("Value27")
.HasColumnType("float");
b.Property<double?>("Value28")
.HasColumnType("float");
b.Property<double?>("Value29")
.HasColumnType("float");
b.Property<double?>("Value3")
.HasColumnType("float");
b.Property<double?>("Value30")
.HasColumnType("float");
b.Property<double?>("Value31")
.HasColumnType("float");
b.Property<double?>("Value32")
.HasColumnType("float");
b.Property<double?>("Value4")
.HasColumnType("float");
b.Property<double?>("Value5")
.HasColumnType("float");
b.Property<double?>("Value6")
.HasColumnType("float");
b.Property<double?>("Value7")
.HasColumnType("float");
b.Property<double?>("Value8")
.HasColumnType("float");
b.Property<double?>("Value9")
.HasColumnType("float");
b.HasKey("Id");
b.HasIndex("CreatedById");
b.HasIndex("LayerId");
b.HasIndex("ModifiedById");
b.ToTable("Records");
});
modelBuilder.Entity("WebAPI.Models.User", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<string>("Email")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.Property<string>("UserName")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.HasKey("Id");
b.ToTable("Users");
});
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.HasOne("WebAPI.Models.User", "CreatedBy")
.WithMany()
.HasForeignKey("CreatedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.User", "ModifiedBy")
.WithMany()
.HasForeignKey("ModifiedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.Layer", "Parent")
.WithMany()
.HasForeignKey("ParentId");
b.Navigation("CreatedBy");
b.Navigation("ModifiedBy");
b.Navigation("Parent");
});
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
{
b.HasOne("WebAPI.Models.Layer", "Source")
.WithMany()
.HasForeignKey("SourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Source");
});
modelBuilder.Entity("WebAPI.Models.Record", b =>
{
b.HasOne("WebAPI.Models.User", "CreatedBy")
.WithMany()
.HasForeignKey("CreatedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.Layer", null)
.WithMany("Records")
.HasForeignKey("LayerId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.User", "ModifiedBy")
.WithMany()
.HasForeignKey("ModifiedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CreatedBy");
b.Navigation("ModifiedBy");
});
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.Navigation("Records");
});
#pragma warning restore 612, 618
}
}
}

40
WebAPI/Migrations/20240825144443_QueueJobs.cs Normal file
View File

@@ -0,0 +1,40 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace WebAPI.Migrations
{
/// <inheritdoc />
public partial class QueueJobs : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "QueueJobs",
columns: table => new
{
Id = table.Column<Guid>(type: "uniqueidentifier", nullable: false),
LayerId = table.Column<Guid>(type: "uniqueidentifier", nullable: false),
Attempts = table.Column<int>(type: "int", nullable: false),
Status = table.Column<int>(type: "int", nullable: false),
Type = table.Column<int>(type: "int", nullable: false),
Message = table.Column<string>(type: "nvarchar(max)", nullable: false),
CreatedAt = table.Column<DateTime>(type: "datetime2", nullable: false),
ModifiedAt = table.Column<DateTime>(type: "datetime2", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_QueueJobs", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "QueueJobs");
}
}
}

33
WebAPI/Migrations/AppDbContextModelSnapshot.cs
View File

@@ -112,6 +112,39 @@ namespace WebAPI.Migrations
b.ToTable("ProcessSources");
});
modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<int>("Attempts")
.HasColumnType("int");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<string>("Message")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<int>("Status")
.HasColumnType("int");
b.Property<int>("Type")
.HasColumnType("int");
b.HasKey("Id");
b.ToTable("QueueJobs");
});
modelBuilder.Entity("WebAPI.Models.Record", b =>
{
b.Property<Guid>("Id")

3
WebAPI/Models/LogEntry.cs
View File

@@ -11,7 +11,8 @@ public enum LogType {
Backup,
Process,
PowerBi,
DataInbox
DataInbox,
Queue
}
public class LogEntry
{

29
WebAPI/Models/QueueJob.cs Normal file
View File

@@ -0,0 +1,29 @@
using System.ComponentModel.DataAnnotations;
using System.Security.Cryptography.X509Certificates;
namespace WebAPI.Models;
public enum JobStatus
{
New,
Failed,
Success
}
public enum JobType
{
ImportWorker,
ProcessWorker
}
public class QueueJob
{
[Key] public Guid Id { get; set; }
[Required] public Guid LayerId { get; set; }
[Required] public int Attempts { get; set; }
[Required] public JobStatus Status { get; set; } = JobStatus.New;
[Required] public JobType Type { get; set; } = JobType.ImportWorker;
public string Message { get; set; } = string.Empty;
[Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
[Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow;
}