@@ -13,7 +13,7 @@ DOCKER_SA_PASSWORD="$&#ojoOOKEJ223"
|
|||||||
DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd"
|
DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd"
|
||||||
DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE}
|
DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE}
|
||||||
#SERVER VARIABLES
|
#SERVER VARIABLES
|
||||||
REMOTE_HOST="crm.bim-it.pl"
|
REMOTE_HOST="bim-it.pl"
|
||||||
REMOTE_USER="mz"
|
REMOTE_USER="mz"
|
||||||
REMOTE_SA_PASSWORD="v](8Lc|RfG"
|
REMOTE_SA_PASSWORD="v](8Lc|RfG"
|
||||||
REMOTE_BACKUP_DIR="/tmp"
|
REMOTE_BACKUP_DIR="/tmp"
|
||||||
|
|||||||
8
Utils/totalCleanup.sh
Normal file
8
Utils/totalCleanup.sh
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
docker stop $(docker ps -aq)
|
||||||
|
docker rm -f $(docker ps -aq)
|
||||||
|
docker rmi -f $(docker images -q)
|
||||||
|
docker network prune -f
|
||||||
|
docker volume prune -f
|
||||||
|
docker builder prune -af
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Database.Context;
|
namespace DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
|
public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
|
||||||
{
|
{
|
||||||
@@ -21,14 +21,4 @@ public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(op
|
|||||||
x.SourceId
|
x.SourceId
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static readonly LoggerFactory MyLoggerFactory =
|
|
||||||
new(new[] {
|
|
||||||
new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider()
|
|
||||||
});
|
|
||||||
|
|
||||||
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
|
|
||||||
{
|
|
||||||
optionsBuilder.UseLoggerFactory(MyLoggerFactory);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -4,24 +4,24 @@ using Microsoft.EntityFrameworkCore;
|
|||||||
using Microsoft.EntityFrameworkCore.Design;
|
using Microsoft.EntityFrameworkCore.Design;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
|
|
||||||
namespace DiunaBI.Database.Context;
|
namespace DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbContext>
|
public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbContext>
|
||||||
{
|
{
|
||||||
public AppDbContext CreateDbContext(string[] args)
|
public AppDbContext CreateDbContext(string[] args)
|
||||||
{
|
{
|
||||||
var configuration = new ConfigurationBuilder()
|
var configuration = new ConfigurationBuilder()
|
||||||
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI")) // ✅ Poprawna ścieżka
|
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI"))
|
||||||
.AddJsonFile("appsettings.json", optional: false)
|
.AddJsonFile("appsettings.json", optional: false)
|
||||||
.AddJsonFile("appsettings.Development.json", optional: true)
|
.AddJsonFile("appsettings.Development.json", optional: true)
|
||||||
.Build();
|
.Build();
|
||||||
|
|
||||||
var optionsBuilder = new DbContextOptionsBuilder<AppDbContext>();
|
var optionsBuilder = new DbContextOptionsBuilder<AppDbContext>();
|
||||||
|
|
||||||
var connectionString = configuration.GetConnectionString("DefaultConnection");
|
var connectionString = configuration.GetConnectionString("SQLDatabase");
|
||||||
if (string.IsNullOrEmpty(connectionString))
|
if (string.IsNullOrEmpty(connectionString))
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException("Connection string 'DefaultConnection' not found in appsettings.json");
|
throw new InvalidOperationException("Connection string 'SQLDatabase' not found in appsettings.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
optionsBuilder.UseSqlServer(connectionString);
|
optionsBuilder.UseSqlServer(connectionString);
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
// <auto-generated />
|
||||||
|
using System;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Metadata;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Migrations
|
||||||
|
{
|
||||||
|
[DbContext(typeof(AppDbContext))]
|
||||||
|
[Migration("20250607084540_QueueJobRefactor")]
|
||||||
|
partial class QueueJobRefactor
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||||
|
{
|
||||||
|
#pragma warning disable 612, 618
|
||||||
|
modelBuilder
|
||||||
|
.HasAnnotation("ProductVersion", "8.0.0")
|
||||||
|
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||||
|
|
||||||
|
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("Data")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(2147483647)
|
||||||
|
.HasColumnType("nvarchar(max)");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<string>("Source")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("DataInbox");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<bool>("IsCancelled")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<bool>("IsDeleted")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<int>("Number")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<Guid?>("ParentId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("Type")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.HasIndex("CreatedById");
|
||||||
|
|
||||||
|
b.HasIndex("ModifiedById");
|
||||||
|
|
||||||
|
b.HasIndex("ParentId");
|
||||||
|
|
||||||
|
b.ToTable("Layers");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<Guid>("SourceId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.HasKey("LayerId", "SourceId");
|
||||||
|
|
||||||
|
b.HasIndex("SourceId");
|
||||||
|
|
||||||
|
b.ToTable("ProcessSources");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("CompletedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("JobType")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("LastAttemptAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("LastError")
|
||||||
|
.HasMaxLength(1000)
|
||||||
|
.HasColumnType("nvarchar(1000)");
|
||||||
|
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("LayerName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(200)
|
||||||
|
.HasColumnType("nvarchar(200)");
|
||||||
|
|
||||||
|
b.Property<int>("MaxRetries")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("PluginName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(100)
|
||||||
|
.HasColumnType("nvarchar(100)");
|
||||||
|
|
||||||
|
b.Property<int>("Priority")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("RetryCount")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("QueueJobs");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Code")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Desc1")
|
||||||
|
.HasMaxLength(10000)
|
||||||
|
.HasColumnType("nvarchar(max)");
|
||||||
|
|
||||||
|
b.Property<bool>("IsDeleted")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<double?>("Value1")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value10")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value11")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value12")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value13")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value14")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value15")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value16")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value17")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value18")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value19")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value2")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value20")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value21")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value22")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value23")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value24")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value25")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value26")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value27")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value28")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value29")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value3")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value30")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value31")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value32")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value4")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value5")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value6")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value7")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value8")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value9")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.HasIndex("CreatedById");
|
||||||
|
|
||||||
|
b.HasIndex("LayerId");
|
||||||
|
|
||||||
|
b.HasIndex("ModifiedById");
|
||||||
|
|
||||||
|
b.ToTable("Records");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("Email")
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<string>("UserName")
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("Users");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("CreatedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ModifiedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ParentId");
|
||||||
|
|
||||||
|
b.Navigation("CreatedBy");
|
||||||
|
|
||||||
|
b.Navigation("ModifiedBy");
|
||||||
|
|
||||||
|
b.Navigation("Parent");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("SourceId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.Navigation("Source");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("CreatedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||||
|
.WithMany("Records")
|
||||||
|
.HasForeignKey("LayerId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ModifiedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.Navigation("CreatedBy");
|
||||||
|
|
||||||
|
b.Navigation("ModifiedBy");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Records");
|
||||||
|
});
|
||||||
|
#pragma warning restore 612, 618
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
using System;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class QueueJobRefactor : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "Message",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Type",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "RetryCount");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "ModifiedAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "ModifiedAtUtc");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Attempts",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Priority");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "CompletedAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "CreatedAtUtc",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<Guid>(
|
||||||
|
name: "CreatedById",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "uniqueidentifier",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<int>(
|
||||||
|
name: "JobType",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "int",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: 0);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "LastAttemptAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "LastError",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(1000)",
|
||||||
|
maxLength: 1000,
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "LayerName",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(200)",
|
||||||
|
maxLength: 200,
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<int>(
|
||||||
|
name: "MaxRetries",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "int",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: 0);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<Guid>(
|
||||||
|
name: "ModifiedById",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "uniqueidentifier",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "PluginName",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(100)",
|
||||||
|
maxLength: 100,
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CompletedAt",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CreatedAtUtc",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CreatedById",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "JobType",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LastAttemptAt",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LastError",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LayerName",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "MaxRetries",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "ModifiedById",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "PluginName",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "RetryCount",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Type");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Priority",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Attempts");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "ModifiedAtUtc",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "ModifiedAt");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "Message",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(max)",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,11 +1,10 @@
|
|||||||
// <auto-generated />
|
// <auto-generated />
|
||||||
using System;
|
using System;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
using Microsoft.EntityFrameworkCore.Metadata;
|
using Microsoft.EntityFrameworkCore.Metadata;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -18,12 +17,12 @@ namespace DiunaBI.Core.Migrations
|
|||||||
{
|
{
|
||||||
#pragma warning disable 612, 618
|
#pragma warning disable 612, 618
|
||||||
modelBuilder
|
modelBuilder
|
||||||
.HasAnnotation("ProductVersion", "9.0.0")
|
.HasAnnotation("ProductVersion", "8.0.0")
|
||||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||||
|
|
||||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.DataInbox", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -52,7 +51,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("DataInbox");
|
b.ToTable("DataInbox");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -101,7 +100,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Layers");
|
b.ToTable("Layers");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("LayerId")
|
b.Property<Guid>("LayerId")
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
@@ -116,32 +115,63 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("ProcessSources");
|
b.ToTable("ProcessSources");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
b.Property<int>("Attempts")
|
b.Property<DateTime?>("CompletedAt")
|
||||||
.HasColumnType("int");
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
b.Property<DateTime>("CreatedAt")
|
b.Property<DateTime>("CreatedAt")
|
||||||
.HasColumnType("datetime2");
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("JobType")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("LastAttemptAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("LastError")
|
||||||
|
.HasMaxLength(1000)
|
||||||
|
.HasColumnType("nvarchar(1000)");
|
||||||
|
|
||||||
b.Property<Guid>("LayerId")
|
b.Property<Guid>("LayerId")
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
b.Property<string>("Message")
|
b.Property<string>("LayerName")
|
||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("nvarchar(max)");
|
.HasMaxLength(200)
|
||||||
|
.HasColumnType("nvarchar(200)");
|
||||||
|
|
||||||
b.Property<DateTime>("ModifiedAt")
|
b.Property<int>("MaxRetries")
|
||||||
.HasColumnType("datetime2");
|
|
||||||
|
|
||||||
b.Property<int>("Status")
|
|
||||||
.HasColumnType("int");
|
.HasColumnType("int");
|
||||||
|
|
||||||
b.Property<int>("Type")
|
b.Property<DateTime>("ModifiedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("PluginName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(100)
|
||||||
|
.HasColumnType("nvarchar(100)");
|
||||||
|
|
||||||
|
b.Property<int>("Priority")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("RetryCount")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
.HasColumnType("int");
|
.HasColumnType("int");
|
||||||
|
|
||||||
b.HasKey("Id");
|
b.HasKey("Id");
|
||||||
@@ -149,7 +179,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("QueueJobs");
|
b.ToTable("QueueJobs");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -289,7 +319,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Records");
|
b.ToTable("Records");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.User", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -311,21 +341,21 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Users");
|
b.ToTable("Users");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("CreatedById")
|
.HasForeignKey("CreatedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ModifiedById")
|
.HasForeignKey("ModifiedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.Layer", "Parent")
|
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ParentId");
|
.HasForeignKey("ParentId");
|
||||||
|
|
||||||
@@ -336,9 +366,9 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("Parent");
|
b.Navigation("Parent");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.Layer", "Source")
|
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("SourceId")
|
.HasForeignKey("SourceId")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
@@ -347,21 +377,21 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("Source");
|
b.Navigation("Source");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("CreatedById")
|
.HasForeignKey("CreatedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.Layer", null)
|
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||||
.WithMany("Records")
|
.WithMany("Records")
|
||||||
.HasForeignKey("LayerId")
|
.HasForeignKey("LayerId")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ModifiedById")
|
.HasForeignKey("ModifiedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
@@ -372,7 +402,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("ModifiedBy");
|
b.Navigation("ModifiedBy");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.Navigation("Records");
|
b.Navigation("Records");
|
||||||
});
|
});
|
||||||
@@ -11,5 +11,19 @@
|
|||||||
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
|
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
|
||||||
<PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
|
<PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.0">
|
||||||
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
</PackageReference>
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.0">
|
||||||
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
|
</PackageReference>
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
@@ -3,27 +3,68 @@ using System.ComponentModel.DataAnnotations;
|
|||||||
|
|
||||||
namespace DiunaBI.Core.Models;
|
namespace DiunaBI.Core.Models;
|
||||||
|
|
||||||
public enum JobStatus
|
public class QueueJob
|
||||||
{
|
{
|
||||||
New,
|
[Key]
|
||||||
Failed,
|
public Guid Id { get; set; } = Guid.NewGuid();
|
||||||
Success
|
|
||||||
|
[Required]
|
||||||
|
public Guid LayerId { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
[MaxLength(200)]
|
||||||
|
public string LayerName { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
[MaxLength(100)]
|
||||||
|
public string PluginName { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public JobType JobType { get; set; }
|
||||||
|
|
||||||
|
public int Priority { get; set; } = 0; // 0 = highest priority
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
||||||
|
|
||||||
|
public int RetryCount { get; set; } = 0;
|
||||||
|
|
||||||
|
public int MaxRetries { get; set; } = 5;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public JobStatus Status { get; set; } = JobStatus.Pending;
|
||||||
|
|
||||||
|
[MaxLength(1000)]
|
||||||
|
public string? LastError { get; set; }
|
||||||
|
|
||||||
|
public DateTime? LastAttemptAt { get; set; }
|
||||||
|
|
||||||
|
public DateTime? CompletedAt { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public Guid CreatedById { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public Guid ModifiedById { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime ModifiedAtUtc { get; set; } = DateTime.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum JobType
|
public enum JobType
|
||||||
{
|
{
|
||||||
ImportWorker,
|
Import = 0,
|
||||||
ProcessWorker
|
Process = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
public class QueueJob
|
public enum JobStatus
|
||||||
{
|
{
|
||||||
[Key] public Guid Id { get; set; }
|
Pending,
|
||||||
[Required] public Guid LayerId { get; set; }
|
Running,
|
||||||
[Required] public int Attempts { get; set; }
|
Completed,
|
||||||
[Required] public JobStatus Status { get; set; } = JobStatus.New;
|
Failed,
|
||||||
[Required] public JobType Type { get; set; } = JobType.ImportWorker;
|
Retrying
|
||||||
public string Message { get; set; } = string.Empty;
|
|
||||||
[Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
|
||||||
[Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow;
|
|
||||||
}
|
}
|
||||||
@@ -45,7 +45,7 @@ public class Record
|
|||||||
public double? Value32 { get; set; }
|
public double? Value32 { get; set; }
|
||||||
//Description fields
|
//Description fields
|
||||||
[StringLength(10000)]
|
[StringLength(10000)]
|
||||||
public string? Desc1 { get; init; }
|
public string? Desc1 { get; set; }
|
||||||
public DateTime CreatedAt { get; set; }
|
public DateTime CreatedAt { get; set; }
|
||||||
public DateTime ModifiedAt { get; set; }
|
public DateTime ModifiedAt { get; set; }
|
||||||
public bool IsDeleted { get; init; }
|
public bool IsDeleted { get; init; }
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
|
|||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
|
|
||||||
namespace DiunaBI.Core.Services;
|
namespace DiunaBI.Core.Services;
|
||||||
|
|
||||||
public static class ProcessHelper
|
public static class ProcessHelper
|
||||||
{
|
{
|
||||||
public static void SetValue(Record record, int number, double? value)
|
public static void SetValue(Record record, int number, double? value)
|
||||||
@@ -184,7 +185,6 @@ public static class ProcessHelper
|
|||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static string GetSheetName(int month, int year)
|
public static string GetSheetName(int month, int year)
|
||||||
{
|
{
|
||||||
if (month < 1 || month > 12)
|
if (month < 1 || month > 12)
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<!-- EF Core packages -->
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.0">
|
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
|
||||||
<PrivateAssets>all</PrivateAssets>
|
|
||||||
</PackageReference>
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.0">
|
|
||||||
<PrivateAssets>all</PrivateAssets>
|
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
|
||||||
</PackageReference>
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
</Project>
|
|
||||||
@@ -12,6 +12,5 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
||||||
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
@@ -1,19 +1,35 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
public class MorskaD1Importer : MorskaBaseImporter
|
public class MorskaD1Importer : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaD1";
|
public override string ImporterType => "Morska.Import.D1";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
private readonly ILogger<MorskaD1Importer> _logger;
|
private readonly ILogger<MorskaD1Importer> _logger;
|
||||||
|
|
||||||
|
// Configuration properties
|
||||||
|
private string? SheetId { get; set; }
|
||||||
|
private string? SheetTabName { get; set; }
|
||||||
|
private string? DataRange { get; set; }
|
||||||
|
private string? ImportYear { get; set; }
|
||||||
|
private string? ImportMonth { get; set; }
|
||||||
|
private string? ImportName { get; set; }
|
||||||
|
private DateTime? StartDate { get; set; }
|
||||||
|
private DateTime? EndDate { get; set; }
|
||||||
|
private bool IsEnabled { get; set; }
|
||||||
|
|
||||||
|
// Cache for sheet data
|
||||||
|
private IList<IList<object>>? _cachedSheetData;
|
||||||
|
private string? _cachedDataKey;
|
||||||
|
|
||||||
public MorskaD1Importer(
|
public MorskaD1Importer(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
@@ -26,47 +42,235 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
try
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
|
||||||
if (sheetId == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
|
// ✅ Clear cache at start
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
|
||||||
|
LoadConfiguration(importWorker);
|
||||||
|
|
||||||
|
if (!ShouldPerformImport(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
ValidateConfiguration();
|
||||||
if (sheetTabName == null)
|
|
||||||
|
PerformImport(importWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// ✅ Clear cache after import
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
// ✅ Dodaj metodę cache
|
||||||
if (year == null)
|
private IList<IList<object>>? GetSheetData()
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||||
|
|
||||||
|
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
return _cachedSheetData;
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
try
|
||||||
if (month == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
|
||||||
|
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||||
|
_cachedSheetData = dataRangeResponse.Values;
|
||||||
|
_cachedDataKey = currentDataKey;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||||
|
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||||
|
|
||||||
|
return _cachedSheetData;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||||
|
ImporterType, SheetId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
private void LoadConfiguration(Layer importWorker)
|
||||||
if (name == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
if (importWorker.Records == null) return;
|
||||||
|
|
||||||
|
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||||
|
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||||
|
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||||
|
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||||
|
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||||
|
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||||
|
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||||
|
|
||||||
|
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||||
|
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||||
|
{
|
||||||
|
StartDate = startDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
|
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||||
if (dataRange == null)
|
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||||
{
|
{
|
||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
EndDate = endDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||||
sheetId, sheetTabName, dataRange);
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool ShouldPerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
if (!IsEnabled)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartDate.HasValue && EndDate.HasValue)
|
||||||
|
{
|
||||||
|
var now = DateTime.UtcNow.Date;
|
||||||
|
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsImportedLayerUpToDate(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||||
|
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||||
|
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||||
|
{
|
||||||
|
var newestLayer = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == importWorker.Id)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (newestLayer == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// ✅ Użyj cache zamiast bezpośredniego API
|
||||||
|
var data = GetSheetData();
|
||||||
|
|
||||||
|
if (data == null || data.Count < 2)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
var isUpToDate = true;
|
||||||
|
|
||||||
|
foreach (var row in data)
|
||||||
|
{
|
||||||
|
if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
|
||||||
|
|
||||||
|
var code = row[0].ToString();
|
||||||
|
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||||
|
|
||||||
|
if (record == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||||
|
ImporterType, code, importWorker.Name);
|
||||||
|
isUpToDate = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check values 3-17 (Value1-Value15)
|
||||||
|
for (int i = 3; i <= 17 && i < row.Count; i++)
|
||||||
|
{
|
||||||
|
var sheetValue = ParseValue(row[i]?.ToString());
|
||||||
|
var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
|
||||||
|
|
||||||
|
if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}",
|
||||||
|
ImporterType, code, i - 2, dbValue, sheetValue);
|
||||||
|
isUpToDate = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||||
|
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||||
|
|
||||||
|
return isUpToDate;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
ImporterType, SheetId, SheetTabName, DataRange);
|
||||||
|
|
||||||
var layer = new Layer
|
var layer = new Layer
|
||||||
{
|
{
|
||||||
@@ -79,17 +283,18 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
CreatedAt = DateTime.UtcNow,
|
CreatedAt = DateTime.UtcNow,
|
||||||
ModifiedAt = DateTime.UtcNow
|
ModifiedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
// ✅ Użyj cache zamiast bezpośredniego API
|
||||||
var data = dataRangeResponse.Values;
|
var data = GetSheetData();
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||||
|
ImporterType, data?.Count ?? 0);
|
||||||
|
|
||||||
var newRecords = (from t in data
|
var newRecords = (from t in data
|
||||||
where t.Count > 1 && (string)t[0] != string.Empty
|
where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
|
||||||
select new Record
|
select new Record
|
||||||
{
|
{
|
||||||
Id = Guid.NewGuid(),
|
Id = Guid.NewGuid(),
|
||||||
@@ -114,20 +319,69 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
}).ToList();
|
}).ToList();
|
||||||
|
|
||||||
_db.Layers.Add(layer);
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
SaveRecords(layer.Id, newRecords);
|
SaveRecords(layer.Id, newRecords);
|
||||||
_db.SaveChanges();
|
_db.SaveChanges();
|
||||||
|
|
||||||
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
newRecords.Count, layer.Name, layer.Id);
|
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
|
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private double? ParseValue(string? value)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
|
||||||
|
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
|
||||||
|
try
|
||||||
|
{
|
||||||
|
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
catch (FormatException e)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}",
|
||||||
|
ImporterType, value, e.Message);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IndexExists(IList<object> array, int index)
|
||||||
|
{
|
||||||
|
return array != null && index >= 0 && index < array.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
private double? GetRecordValueByIndex(Record record, int valueIndex)
|
||||||
|
{
|
||||||
|
return valueIndex switch
|
||||||
|
{
|
||||||
|
1 => record.Value1,
|
||||||
|
2 => record.Value2,
|
||||||
|
3 => record.Value3,
|
||||||
|
4 => record.Value4,
|
||||||
|
5 => record.Value5,
|
||||||
|
6 => record.Value6,
|
||||||
|
7 => record.Value7,
|
||||||
|
8 => record.Value8,
|
||||||
|
9 => record.Value9,
|
||||||
|
10 => record.Value10,
|
||||||
|
11 => record.Value11,
|
||||||
|
12 => record.Value12,
|
||||||
|
13 => record.Value13,
|
||||||
|
14 => record.Value14,
|
||||||
|
15 => record.Value15,
|
||||||
|
_ => null
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
{
|
{
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
@@ -146,27 +400,7 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
_db.Records.Add(record);
|
_db.Records.Add(record);
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
}
|
ImporterType, records.Count, layerId);
|
||||||
|
|
||||||
private double? ParseValue(string? value)
|
|
||||||
{
|
|
||||||
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
|
|
||||||
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
|
|
||||||
try
|
|
||||||
{
|
|
||||||
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
catch (FormatException e)
|
|
||||||
{
|
|
||||||
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private bool IndexExists(IList<object> array, int index)
|
|
||||||
{
|
|
||||||
return array != null && index >= 0 && index < array.Count;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2,18 +2,28 @@
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
public class MorskaD3Importer : MorskaBaseImporter
|
public class MorskaD3Importer : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaD3";
|
public override string ImporterType => "Morska.Import.D3";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly ILogger<MorskaD3Importer> _logger;
|
private readonly ILogger<MorskaD3Importer> _logger;
|
||||||
|
|
||||||
|
// Configuration properties
|
||||||
|
private string? ImportYear { get; set; }
|
||||||
|
private string? ImportMonth { get; set; }
|
||||||
|
private string? ImportName { get; set; }
|
||||||
|
private string? ImportType { get; set; }
|
||||||
|
private DateTime? StartDate { get; set; }
|
||||||
|
private DateTime? EndDate { get; set; }
|
||||||
|
private bool IsEnabled { get; set; }
|
||||||
|
|
||||||
public MorskaD3Importer(
|
public MorskaD3Importer(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
ILogger<MorskaD3Importer> logger)
|
ILogger<MorskaD3Importer> logger)
|
||||||
@@ -24,67 +34,191 @@ public class MorskaD3Importer : MorskaBaseImporter
|
|||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
try
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
|
||||||
if (year == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
|
LoadConfiguration(importWorker);
|
||||||
|
|
||||||
|
if (!ShouldPerformImport(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
ValidateConfiguration();
|
||||||
if (month == null)
|
|
||||||
|
PerformImport(importWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
private void LoadConfiguration(Layer importWorker)
|
||||||
if (name == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
if (importWorker.Records == null) return;
|
||||||
|
|
||||||
|
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||||
|
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||||
|
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||||
|
ImportType = GetRecordValue(importWorker.Records, "ImportType");
|
||||||
|
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||||
|
|
||||||
|
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||||
|
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||||
|
{
|
||||||
|
StartDate = startDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
|
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||||
if (type == null)
|
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportType not found, {importWorker.Name}");
|
EndDate = endDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
|
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}",
|
||||||
|
ImporterType, importWorker.Name, ImportType);
|
||||||
|
}
|
||||||
|
|
||||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
|
private bool ShouldPerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
if (!IsEnabled)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartDate.HasValue && EndDate.HasValue)
|
||||||
|
{
|
||||||
|
var now = DateTime.UtcNow.Date;
|
||||||
|
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsImportedLayerUpToDate(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||||
|
{
|
||||||
|
var newestLayer = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == importWorker.Id)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (newestLayer == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||||
if (dataInbox == null)
|
if (dataInbox == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"DataInbox not found, {type}");
|
_logger.LogWarning("{ImporterType}: No DataInbox found for type {ImportType}",
|
||||||
|
ImporterType, ImportType);
|
||||||
|
return true; // Assume up to date if no data source
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
// Compare timestamps - if DataInbox is newer than our layer, we need to import
|
||||||
dataInbox.Id, dataInbox.CreatedAt);
|
var isUpToDate = newestLayer.CreatedAt >= dataInbox.CreatedAt;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Layer created at {LayerTime}, DataInbox created at {DataTime}, up to date: {IsUpToDate}",
|
||||||
|
ImporterType, newestLayer.CreatedAt, dataInbox.CreatedAt, isUpToDate);
|
||||||
|
|
||||||
|
return isUpToDate;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Looking for DataInbox with type {ImportType}",
|
||||||
|
ImporterType, ImportType);
|
||||||
|
|
||||||
|
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||||
|
if (dataInbox == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||||
|
ImporterType, dataInbox.Id, dataInbox.CreatedAt);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var data = Convert.FromBase64String(dataInbox.Data);
|
var data = Convert.FromBase64String(dataInbox.Data);
|
||||||
var jsonString = Encoding.UTF8.GetString(data);
|
var jsonString = Encoding.UTF8.GetString(data);
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
|
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
|
||||||
|
ImporterType, data.Length);
|
||||||
|
|
||||||
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
||||||
if (records == null)
|
if (records == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
|
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
|
_logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
|
||||||
|
ImporterType, records.Count);
|
||||||
|
|
||||||
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
|
records = records.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
|
||||||
if (records.Count == 0)
|
if (records.Count == 0)
|
||||||
{
|
{
|
||||||
throw new Exception($"No records found for {year}{month}");
|
throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
|
_logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||||
records.Count, year, month);
|
ImporterType, records.Count, ImportYear, ImportMonth);
|
||||||
|
|
||||||
records = records.Select(x =>
|
records = records.Select(x =>
|
||||||
{
|
{
|
||||||
@@ -105,23 +239,28 @@ public class MorskaD3Importer : MorskaBaseImporter
|
|||||||
CreatedAt = DateTime.UtcNow,
|
CreatedAt = DateTime.UtcNow,
|
||||||
ModifiedAt = DateTime.UtcNow
|
ModifiedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
_db.Layers.Add(layer);
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
SaveRecords(layer.Id, records);
|
SaveRecords(layer.Id, records);
|
||||||
_db.SaveChanges();
|
_db.SaveChanges();
|
||||||
|
|
||||||
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
records.Count, layer.Name, layer.Id);
|
ImporterType, records.Count, layer.Name, layer.Id);
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
|
_logger.LogError(e, "{ImporterType}: Error processing DataInbox {DataInboxId}",
|
||||||
|
ImporterType, dataInbox.Id);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
{
|
{
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
@@ -140,6 +279,7 @@ public class MorskaD3Importer : MorskaBaseImporter
|
|||||||
_db.Records.Add(record);
|
_db.Records.Add(record);
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ImporterType, records.Count, layerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,19 +1,35 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
public class MorskaFk2Importer : MorskaBaseImporter
|
public class MorskaFk2Importer : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaFK2";
|
public override string ImporterType => "Morska.Import.FK2";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
private readonly ILogger<MorskaFk2Importer> _logger;
|
private readonly ILogger<MorskaFk2Importer> _logger;
|
||||||
|
|
||||||
|
// Configuration properties
|
||||||
|
private string? SheetId { get; set; }
|
||||||
|
private string? SheetTabName { get; set; }
|
||||||
|
private string? DataRange { get; set; }
|
||||||
|
private string? ImportYear { get; set; }
|
||||||
|
private string? ImportMonth { get; set; }
|
||||||
|
private string? ImportName { get; set; }
|
||||||
|
private DateTime? StartDate { get; set; }
|
||||||
|
private DateTime? EndDate { get; set; }
|
||||||
|
private bool IsEnabled { get; set; }
|
||||||
|
|
||||||
|
// Cache for sheet data
|
||||||
|
private IList<IList<object>>? _cachedSheetData;
|
||||||
|
private string? _cachedDataKey;
|
||||||
|
|
||||||
public MorskaFk2Importer(
|
public MorskaFk2Importer(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
@@ -26,47 +42,250 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
_logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
try
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
|
||||||
if (sheetId == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
|
// ✅ Clear cache at start
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
|
||||||
|
LoadConfiguration(importWorker);
|
||||||
|
|
||||||
|
if (!ShouldPerformImport(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
ValidateConfiguration();
|
||||||
if (sheetTabName == null)
|
|
||||||
|
PerformImport(importWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// ✅ Clear cache after import
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
// ✅ Dodaj metodę cache
|
||||||
if (year == null)
|
private IList<IList<object>>? GetSheetData()
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||||
|
|
||||||
|
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
return _cachedSheetData;
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
try
|
||||||
if (month == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
|
||||||
|
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||||
|
_cachedSheetData = dataRangeResponse.Values;
|
||||||
|
_cachedDataKey = currentDataKey;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||||
|
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||||
|
|
||||||
|
return _cachedSheetData;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||||
|
ImporterType, SheetId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
private void LoadConfiguration(Layer importWorker)
|
||||||
if (name == null)
|
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
if (importWorker.Records == null) return;
|
||||||
|
|
||||||
|
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||||
|
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||||
|
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||||
|
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||||
|
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||||
|
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||||
|
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||||
|
|
||||||
|
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||||
|
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||||
|
{
|
||||||
|
StartDate = startDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
|
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||||
if (dataRange == null)
|
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||||
{
|
{
|
||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
EndDate = endDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||||
sheetId, sheetTabName, dataRange);
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool ShouldPerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
if (!IsEnabled)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartDate.HasValue && EndDate.HasValue)
|
||||||
|
{
|
||||||
|
var now = DateTime.UtcNow.Date;
|
||||||
|
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!IsImportedLayerUpToDate(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||||
|
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||||
|
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||||
|
{
|
||||||
|
var newestLayer = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == importWorker.Id)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (newestLayer == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// ✅ Użyj cache zamiast bezpośredniego API
|
||||||
|
var data = GetSheetData();
|
||||||
|
|
||||||
|
if (data == null || data.Count == 0)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
var isUpToDate = true;
|
||||||
|
|
||||||
|
for (var i = 0; i < data.Count; i++)
|
||||||
|
{
|
||||||
|
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var dateArr = data[i][1].ToString()!.Split(".");
|
||||||
|
if (dateArr.Length != 3) continue;
|
||||||
|
|
||||||
|
var number = data[i][2].ToString()!;
|
||||||
|
if (number.Length == 1) number = $"0{number}";
|
||||||
|
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||||
|
|
||||||
|
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||||
|
if (record == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||||
|
ImporterType, code, importWorker.Name);
|
||||||
|
isUpToDate = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) &&
|
||||||
|
Math.Abs((double)(record.Value1 - value)!) >= 0.01)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||||
|
ImporterType, code, record.Value1, value);
|
||||||
|
isUpToDate = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (record.Desc1 != data[i][3]?.ToString())
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}",
|
||||||
|
ImporterType, code, record.Desc1, data[i][3]?.ToString());
|
||||||
|
isUpToDate = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison",
|
||||||
|
ImporterType, i);
|
||||||
|
isUpToDate = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||||
|
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||||
|
|
||||||
|
return isUpToDate;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
ImporterType, SheetId, SheetTabName, DataRange);
|
||||||
|
|
||||||
var layer = new Layer
|
var layer = new Layer
|
||||||
{
|
{
|
||||||
@@ -79,16 +298,17 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
CreatedAt = DateTime.UtcNow,
|
CreatedAt = DateTime.UtcNow,
|
||||||
ModifiedAt = DateTime.UtcNow
|
ModifiedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
// ✅ Użyj cache zamiast bezpośredniego API
|
||||||
var data = dataRangeResponse.Values;
|
var data = GetSheetData();
|
||||||
|
|
||||||
_logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||||
|
ImporterType, data?.Count ?? 0);
|
||||||
|
|
||||||
if (data != null)
|
if (data != null)
|
||||||
{
|
{
|
||||||
@@ -96,15 +316,17 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
{
|
{
|
||||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
||||||
{
|
{
|
||||||
_logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i);
|
_logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc",
|
||||||
|
ImporterType, i);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
var dateArr = data[i][1].ToString()!.Split(".");
|
var dateArr = data[i][1].ToString()!.Split(".");
|
||||||
if (dateArr.Length != 3)
|
if (dateArr.Length != 3)
|
||||||
{
|
{
|
||||||
_logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]);
|
_logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}",
|
||||||
throw new Exception($"Invalid date in row {i}");
|
ImporterType, i, data[i][1]);
|
||||||
|
throw new InvalidOperationException($"Invalid date in row {i}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var number = data[i][2].ToString()!;
|
var number = data[i][2].ToString()!;
|
||||||
@@ -114,7 +336,8 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
||||||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||||
{
|
{
|
||||||
_logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i);
|
_logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value",
|
||||||
|
ImporterType, i);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,20 +355,24 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
}
|
}
|
||||||
|
|
||||||
_db.Layers.Add(layer);
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
SaveRecords(layer.Id, newRecords);
|
SaveRecords(layer.Id, newRecords);
|
||||||
_db.SaveChanges();
|
_db.SaveChanges();
|
||||||
|
|
||||||
_logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
newRecords.Count, layer.Name, layer.Id);
|
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
_logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId);
|
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
{
|
{
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
@@ -164,6 +391,7 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
_db.Records.Add(record);
|
_db.Records.Add(record);
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ImporterType, records.Count, layerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
using System.Globalization;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
|
||||||
|
|
||||||
public class MorskaImporter : MorskaBaseImporter
|
|
||||||
{
|
|
||||||
public override string ImporterType => "MorskaImporter";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<MorskaImporter> _logger;
|
|
||||||
|
|
||||||
public MorskaImporter(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<MorskaImporter> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Import(Layer importWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
|
||||||
if (sheetId == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
|
||||||
if (sheetTabName == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
|
||||||
if (year == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
|
||||||
if (month == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
|
||||||
if (name == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
|
|
||||||
if (dataRange == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
|
||||||
sheetId, sheetTabName, dataRange);
|
|
||||||
|
|
||||||
var layer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Number = _db.Layers.Count() + 1,
|
|
||||||
ParentId = importWorker.Id,
|
|
||||||
Type = LayerType.Import,
|
|
||||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
||||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
|
||||||
var data = dataRangeResponse.Values;
|
|
||||||
|
|
||||||
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
|
||||||
|
|
||||||
if (data != null && data.Count >= 2)
|
|
||||||
{
|
|
||||||
for (var i = 0; i < data[1].Count; i++)
|
|
||||||
{
|
|
||||||
if (!(data[0][i].ToString()?.Length > 0) ||
|
|
||||||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
|
||||||
{
|
|
||||||
_logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var record = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = data[0][i].ToString(),
|
|
||||||
Value1 = value,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
newRecords.Add(record);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_db.Layers.Add(layer);
|
|
||||||
|
|
||||||
SaveRecords(layer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
|
||||||
newRecords.Count, layer.Name, layer.Id);
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,359 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
|
public class MorskaStandardImporter : MorskaBaseImporter
|
||||||
|
{
|
||||||
|
public override string ImporterType => "Morska.Import.Standard";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaStandardImporter> _logger;
|
||||||
|
|
||||||
|
// Configuration properties
|
||||||
|
private string? SheetId { get; set; }
|
||||||
|
private string? SheetTabName { get; set; }
|
||||||
|
private string? DataRange { get; set; }
|
||||||
|
private string? ImportYear { get; set; }
|
||||||
|
private string? ImportMonth { get; set; }
|
||||||
|
private string? ImportName { get; set; }
|
||||||
|
private DateTime? StartDate { get; set; }
|
||||||
|
private DateTime? EndDate { get; set; }
|
||||||
|
private bool IsEnabled { get; set; }
|
||||||
|
|
||||||
|
private IList<IList<object>>? _cachedSheetData;
|
||||||
|
private string? _cachedDataKey;
|
||||||
|
|
||||||
|
public MorskaStandardImporter(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaStandardImporter> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Import(Layer importWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
|
// Clear cache before import
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(importWorker);
|
||||||
|
|
||||||
|
// Check if import should be performed
|
||||||
|
if (!ShouldPerformImport(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual import
|
||||||
|
PerformImport(importWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
ImporterType, importWorker.Name, importWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
// Clear cache after import
|
||||||
|
_cachedSheetData = null;
|
||||||
|
_cachedDataKey = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private IList<IList<object>>? GetSheetData()
|
||||||
|
{
|
||||||
|
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||||
|
|
||||||
|
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
return _cachedSheetData;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||||
|
ImporterType, currentDataKey);
|
||||||
|
|
||||||
|
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||||
|
_cachedSheetData = dataRangeResponse.Values;
|
||||||
|
_cachedDataKey = currentDataKey;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||||
|
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||||
|
|
||||||
|
return _cachedSheetData;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||||
|
ImporterType, SheetId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer importWorker)
|
||||||
|
{
|
||||||
|
if (importWorker.Records == null) return;
|
||||||
|
|
||||||
|
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||||
|
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||||
|
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||||
|
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||||
|
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||||
|
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||||
|
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||||
|
|
||||||
|
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||||
|
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||||
|
{
|
||||||
|
StartDate = startDate;
|
||||||
|
}
|
||||||
|
|
||||||
|
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||||
|
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||||
|
{
|
||||||
|
EndDate = endDate;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool ShouldPerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
if (!IsEnabled)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check date range if configured
|
||||||
|
if (StartDate.HasValue && EndDate.HasValue)
|
||||||
|
{
|
||||||
|
var now = DateTime.UtcNow.Date;
|
||||||
|
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Outside date range - check if imported layer is up to date
|
||||||
|
if (!IsImportedLayerUpToDate(importWorker))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// No date constraints - always import
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||||
|
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||||
|
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||||
|
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||||
|
{
|
||||||
|
var newestLayer = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == importWorker.Id)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (newestLayer == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var data = GetSheetData();
|
||||||
|
|
||||||
|
if (data == null || data.Count < 2)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the number of columns matches
|
||||||
|
if (data[0].Count != data[1].Count)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = 0; i < data[1].Count; i++)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||||
|
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||||
|
ImporterType, i);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
|
||||||
|
if (existingRecord == null || existingRecord.Value1 != value)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}",
|
||||||
|
ImporterType, data[0][i]);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
|
||||||
|
ImporterType, importWorker.Name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformImport(Layer importWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
ImporterType, SheetId, SheetTabName, DataRange);
|
||||||
|
|
||||||
|
var layer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
ParentId = importWorker.Id,
|
||||||
|
Type = LayerType.Import,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var data = GetSheetData();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||||
|
ImporterType, data?.Count ?? 0);
|
||||||
|
|
||||||
|
if (data != null && data.Count >= 2)
|
||||||
|
{
|
||||||
|
for (var i = 0; i < data[1].Count; i++)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||||
|
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||||
|
ImporterType, i);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var record = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = data[0][i].ToString(),
|
||||||
|
Value1 = value,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
newRecords.Add(record);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_db.Layers.Add(layer);
|
||||||
|
SaveRecords(layer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ImporterType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,496 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Google.Apis.Sheets.v4.Data;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using DiunaBI.Core.Services.Calculations;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT1R1Processor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T1.R1";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT1R1Processor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private List<Record>? Sources { get; set; }
|
||||||
|
private List<Record>? DynamicCodes { get; set; }
|
||||||
|
private string? GoogleSheetName { get; set; }
|
||||||
|
|
||||||
|
public MorskaT1R1Processor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT1R1Processor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load sources
|
||||||
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||||
|
if (Sources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source records not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load dynamic codes
|
||||||
|
DynamicCodes = processWorker.Records
|
||||||
|
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||||
|
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
// Load Google Sheet name
|
||||||
|
GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
|
||||||
|
if (string.IsNullOrEmpty(GoogleSheetName))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("GoogleSheetName record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}",
|
||||||
|
ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||||
|
if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
|
||||||
|
ProcessorType, Year, Sources!.Count);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Process records for each month
|
||||||
|
var newRecords = ProcessAllMonths();
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
// Update Google Sheet report
|
||||||
|
UpdateGoogleSheetReport(processedLayer.Id);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessAllMonths()
|
||||||
|
{
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
for (var month = 1; month < 14; month++)
|
||||||
|
{
|
||||||
|
// Skip future months (except month 13 which is summary)
|
||||||
|
if (Year > DateTime.UtcNow.Year ||
|
||||||
|
(Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}",
|
||||||
|
ProcessorType, Year, month);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}",
|
||||||
|
ProcessorType, month, Year);
|
||||||
|
|
||||||
|
var monthRecords = ProcessSingleMonth(month);
|
||||||
|
newRecords.AddRange(monthRecords);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}",
|
||||||
|
ProcessorType, monthRecords.Count, month);
|
||||||
|
}
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessSingleMonth(int month)
|
||||||
|
{
|
||||||
|
var records = new List<Record>();
|
||||||
|
|
||||||
|
// Collect records from all sources
|
||||||
|
foreach (var source in Sources!)
|
||||||
|
{
|
||||||
|
var sourceRecords = GetSourceRecords(source, month);
|
||||||
|
records.AddRange(sourceRecords);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process dynamic codes (calculations)
|
||||||
|
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||||
|
{
|
||||||
|
var calculatedRecords = ProcessDynamicCodes(records, month);
|
||||||
|
records.AddRange(calculatedRecords);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create final records with month suffix
|
||||||
|
var monthRecords = records.Select(x => new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"{x.Code}{month:D2}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow,
|
||||||
|
Value1 = x.Value32
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
return monthRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> GetSourceRecords(Record source, int month)
|
||||||
|
{
|
||||||
|
var dataSource = _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSource == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
var sourceRecords = new List<Record>();
|
||||||
|
|
||||||
|
// Check if there are specific codes configured for this source
|
||||||
|
var codesRecord = Sources!
|
||||||
|
.Where(x => x.Code == $"Codes-{source.Desc1}")
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (codesRecord != null)
|
||||||
|
{
|
||||||
|
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
|
||||||
|
sourceRecords.AddRange(dataSource.Records!
|
||||||
|
.Where(x => codes.Contains(int.Parse(x.Code!))));
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes",
|
||||||
|
ProcessorType, source.Desc1, codes.Count);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
sourceRecords.AddRange(dataSource.Records!);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records",
|
||||||
|
ProcessorType, source.Desc1, dataSource.Records!.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sourceRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessDynamicCodes(List<Record> records, int month)
|
||||||
|
{
|
||||||
|
var calculatedRecords = new List<Record>();
|
||||||
|
|
||||||
|
foreach (var dynamicCode in DynamicCodes!)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}",
|
||||||
|
ProcessorType, dynamicCode.Id, month);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||||
|
if (!calc.IsFormulaCorrect())
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}",
|
||||||
|
ProcessorType, calc.Expression, dynamicCode.Id, month);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calculatedRecord = calc.CalculateT1(records);
|
||||||
|
calculatedRecords.Add(calculatedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}",
|
||||||
|
ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}",
|
||||||
|
ProcessorType, dynamicCode.Desc1, month);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return calculatedRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}",
|
||||||
|
ProcessorType, GoogleSheetName);
|
||||||
|
|
||||||
|
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
|
||||||
|
|
||||||
|
// Get processed layer data
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.Id == sourceId)
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get codes from sheet header
|
||||||
|
var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute();
|
||||||
|
var codesRow = codesResponse.Values[0];
|
||||||
|
|
||||||
|
// Update monthly data (months 1-12)
|
||||||
|
UpdateMonthlyData(sheetId, codesRow, processedLayer);
|
||||||
|
|
||||||
|
// Update summary row (month 13)
|
||||||
|
UpdateSummaryData(sheetId, codesRow, processedLayer);
|
||||||
|
|
||||||
|
// Update timestamps
|
||||||
|
UpdateTimestamps(sheetId, processedLayer);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}",
|
||||||
|
ProcessorType, GoogleSheetName);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}",
|
||||||
|
ProcessorType, GoogleSheetName);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateMonthlyData(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||||
|
{
|
||||||
|
var valueRange = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>>()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Process months 1-12
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
var monthValues = new List<object>();
|
||||||
|
foreach (string code in codesRow)
|
||||||
|
{
|
||||||
|
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}");
|
||||||
|
monthValues.Add(record?.Value1?.ToString() ?? "0");
|
||||||
|
}
|
||||||
|
valueRange.Values.Add(monthValues);
|
||||||
|
}
|
||||||
|
|
||||||
|
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18");
|
||||||
|
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
update.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateSummaryData(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||||
|
{
|
||||||
|
var valueRange = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>>()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Empty row
|
||||||
|
var emptyRow = new List<object>();
|
||||||
|
for (int i = 0; i < codesRow.Count; i++)
|
||||||
|
{
|
||||||
|
emptyRow.Add("");
|
||||||
|
}
|
||||||
|
valueRange.Values.Add(emptyRow);
|
||||||
|
|
||||||
|
// Summary row (month 13)
|
||||||
|
var summaryValues = new List<object>();
|
||||||
|
foreach (string code in codesRow)
|
||||||
|
{
|
||||||
|
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
||||||
|
summaryValues.Add(record?.Value1?.ToString() ?? "0");
|
||||||
|
}
|
||||||
|
valueRange.Values.Add(summaryValues);
|
||||||
|
|
||||||
|
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20");
|
||||||
|
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
update.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateTimestamps(string sheetId, Layer processedLayer)
|
||||||
|
{
|
||||||
|
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
|
||||||
|
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||||
|
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||||
|
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
|
||||||
|
var valueRangeTime = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>>
|
||||||
|
{
|
||||||
|
new List<object> { timeUtc },
|
||||||
|
new List<object> { timeWarsaw }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2");
|
||||||
|
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateTime.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||||
|
ProcessorType, timeUtc, timeWarsaw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,467 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Google.Apis.Sheets.v4.Data;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT1R3Processor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T1.R3";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT1R3Processor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private string? Source { get; set; }
|
||||||
|
|
||||||
|
public MorskaT1R3Processor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT1R3Processor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load source
|
||||||
|
Source = GetRecordValue(processWorker.Records, "Source");
|
||||||
|
if (string.IsNullOrEmpty(Source))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Source);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Source);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
// Update Google Sheet report
|
||||||
|
UpdateGoogleSheetReport(processedLayer.Id);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$";
|
||||||
|
|
||||||
|
var dataSources = _db.Layers
|
||||||
|
.Where(x => !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.Include(layer => layer.Records!)
|
||||||
|
.AsNoTracking()
|
||||||
|
.AsEnumerable()
|
||||||
|
.Where(x => Regex.IsMatch(x.Name!, pattern))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for pattern: {pattern}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}",
|
||||||
|
ProcessorType, dataSources.Count, pattern);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
foreach (var dataSource in dataSources)
|
||||||
|
{
|
||||||
|
var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!);
|
||||||
|
if (monthStr == null || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}",
|
||||||
|
ProcessorType, dataSource.Name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}",
|
||||||
|
ProcessorType, dataSource.Name, month);
|
||||||
|
|
||||||
|
var sourceRecords = ProcessDataSourceRecords(dataSource, month);
|
||||||
|
newRecords.AddRange(sourceRecords);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}",
|
||||||
|
ProcessorType, sourceRecords.Count, dataSource.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessDataSourceRecords(Layer dataSource, int month)
|
||||||
|
{
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
foreach (var record in dataSource.Records!)
|
||||||
|
{
|
||||||
|
if (record.Value1 == null)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null",
|
||||||
|
ProcessorType, record.Code);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process values for positions 1-32
|
||||||
|
for (var i = 1; i < 33; i++)
|
||||||
|
{
|
||||||
|
var value = ProcessHelper.GetValue(record, i);
|
||||||
|
if (value == null)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var baseValue = (double)record.Value1!;
|
||||||
|
var positionValue = (double)value;
|
||||||
|
var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100;
|
||||||
|
|
||||||
|
var newRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"{record.Code}{month:D2}{i:D2}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow,
|
||||||
|
Value1 = calculatedValue,
|
||||||
|
Desc1 = record.Desc1
|
||||||
|
};
|
||||||
|
|
||||||
|
newRecords.Add(newRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}",
|
||||||
|
ProcessorType, newRecord.Code, newRecord.Value1, record.Code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}",
|
||||||
|
ProcessorType, sourceId);
|
||||||
|
|
||||||
|
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";
|
||||||
|
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.Id == sourceId)
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update sheets for all months
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
UpdateMonthSheet(sheetId, processedLayer, month);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months",
|
||||||
|
ProcessorType);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}",
|
||||||
|
ProcessorType, sourceId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month)
|
||||||
|
{
|
||||||
|
var sheetName = ProcessHelper.GetSheetName(month, Year);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}",
|
||||||
|
ProcessorType, sheetName, month);
|
||||||
|
|
||||||
|
// Get codes from sheet
|
||||||
|
ValueRange? dataRangeResponse;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute();
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}",
|
||||||
|
ProcessorType, sheetName, e.Message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataRangeResponse?.Values == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping",
|
||||||
|
ProcessorType, sheetName);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update data
|
||||||
|
UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month);
|
||||||
|
|
||||||
|
// Update timestamps
|
||||||
|
UpdateSheetTimestamps(sheetId, sheetName, processedLayer);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}",
|
||||||
|
ProcessorType, sheetName);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}",
|
||||||
|
ProcessorType, sheetName, month);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList<IList<object>> codeRows, int month)
|
||||||
|
{
|
||||||
|
var updateValueRange = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>>()
|
||||||
|
};
|
||||||
|
|
||||||
|
foreach (var row in codeRows)
|
||||||
|
{
|
||||||
|
if (row.Count == 0) continue;
|
||||||
|
|
||||||
|
var code = row[0].ToString();
|
||||||
|
var updateRow = new List<object>();
|
||||||
|
|
||||||
|
// Process columns C to Q (positions 1-15)
|
||||||
|
for (var position = 1; position <= 15; position++)
|
||||||
|
{
|
||||||
|
var recordCode = $"{code}{month:D2}{position:D2}";
|
||||||
|
var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode);
|
||||||
|
|
||||||
|
if (codeRecord?.Value1 != null)
|
||||||
|
{
|
||||||
|
updateRow.Add(codeRecord.Value1);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}",
|
||||||
|
ProcessorType, codeRecord.Value1, recordCode);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
updateRow.Add("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
updateValueRange.Values.Add(updateRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update sheet with new values
|
||||||
|
var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200");
|
||||||
|
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
update.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}",
|
||||||
|
ProcessorType, updateValueRange.Values.Count, sheetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer)
|
||||||
|
{
|
||||||
|
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
|
||||||
|
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||||
|
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||||
|
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
|
||||||
|
var valueRangeTime = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>>
|
||||||
|
{
|
||||||
|
new List<object> { timeUtc },
|
||||||
|
new List<object> { timeWarsaw }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
|
||||||
|
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateTime.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||||
|
ProcessorType, sheetName, timeUtc, timeWarsaw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,310 @@
|
|||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "T3.MultiSourceCopySelectedCodes";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaT3MultiSourceCopySelectedCodesProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private int Month { get; set; }
|
||||||
|
private List<Record>? Sources { get; set; }
|
||||||
|
private string? Codes { get; set; }
|
||||||
|
private List<int>? CodesList { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3MultiSourceCopySelectedCodesProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
ILogger<MorskaT3MultiSourceCopySelectedCodesProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load month
|
||||||
|
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||||
|
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Month record not found or invalid");
|
||||||
|
}
|
||||||
|
Month = month;
|
||||||
|
|
||||||
|
// Load sources
|
||||||
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||||
|
if (Sources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source records not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load codes
|
||||||
|
Codes = GetRecordValue(processWorker.Records, "Codes");
|
||||||
|
if (string.IsNullOrEmpty(Codes))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Codes record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse codes list
|
||||||
|
CodesList = ProcessHelper.ParseCodes(Codes);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}",
|
||||||
|
ProcessorType, Year, Month, Sources.Count, CodesList.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||||
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||||
|
if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
|
||||||
|
ProcessorType, Year, Month, Sources!.Count);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = new List<Layer>();
|
||||||
|
|
||||||
|
foreach (var source in Sources!)
|
||||||
|
{
|
||||||
|
var dataSource = _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSource == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3",
|
||||||
|
ProcessorType, Year, Month, source.Desc1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
dataSources.Add(dataSource);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
|
||||||
|
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources",
|
||||||
|
ProcessorType, dataSources.Count);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
|
||||||
|
var filteredRecords = allSourceRecords
|
||||||
|
.Where(x => !string.IsNullOrEmpty(x.Code) &&
|
||||||
|
int.TryParse(x.Code, out var code) &&
|
||||||
|
CodesList!.Contains(code))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes",
|
||||||
|
ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count);
|
||||||
|
|
||||||
|
var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Record CreateCopiedRecord(Record sourceRecord)
|
||||||
|
{
|
||||||
|
var newRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = sourceRecord.Code,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Copy all values from positions 1-32
|
||||||
|
for (var i = 1; i < 33; i++)
|
||||||
|
{
|
||||||
|
var value = ProcessHelper.GetValue(sourceRecord, i);
|
||||||
|
ProcessHelper.SetValue(newRecord, i, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32",
|
||||||
|
ProcessorType, newRecord.Code);
|
||||||
|
|
||||||
|
return newRecord;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
ILogger<MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}",
|
||||||
|
ProcessorType, Year);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}",
|
||||||
|
ProcessorType, Year);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources for all months
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records (sum all monthly values)
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = new List<Layer>();
|
||||||
|
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
var dataSource = _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSource != null)
|
||||||
|
{
|
||||||
|
dataSources.Add(dataSource);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
|
||||||
|
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
|
||||||
|
ProcessorType, month);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for year {Year}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
|
||||||
|
ProcessorType, dataSources.Count, Year);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
var baseRecords = dataSources.Last().Records!;
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
|
||||||
|
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
|
||||||
|
|
||||||
|
foreach (var baseRecord in baseRecords)
|
||||||
|
{
|
||||||
|
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||||
|
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = baseRecord.Code,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sum values from all months for positions 1-32
|
||||||
|
for (var position = 1; position <= 32; position++)
|
||||||
|
{
|
||||||
|
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0);
|
||||||
|
ProcessHelper.SetValue(processedRecord, position, totalValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
|
||||||
|
ProcessorType, baseRecord.Code, codeRecords.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,340 @@
|
|||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services.Calculations;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3MultiSourceSummaryProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T3.MultiSourceSummary";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaT3MultiSourceSummaryProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private int Month { get; set; }
|
||||||
|
private List<Record>? Sources { get; set; }
|
||||||
|
private List<Record>? DynamicCodes { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3MultiSourceSummaryProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
ILogger<MorskaT3MultiSourceSummaryProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load month
|
||||||
|
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||||
|
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Month record not found or invalid");
|
||||||
|
}
|
||||||
|
Month = month;
|
||||||
|
|
||||||
|
// Load sources
|
||||||
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||||
|
if (Sources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source records not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load dynamic codes
|
||||||
|
DynamicCodes = processWorker.Records
|
||||||
|
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||||
|
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
|
||||||
|
ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||||
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
|
||||||
|
ProcessorType, Year, Month, Sources!.Count);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records (sum by base codes)
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Process dynamic codes if configured
|
||||||
|
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||||
|
{
|
||||||
|
var calculatedRecords = ProcessDynamicCodes(newRecords);
|
||||||
|
newRecords.AddRange(calculatedRecords);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = Sources!.Select(source =>
|
||||||
|
_db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault())
|
||||||
|
.OfType<Layer>()
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
|
||||||
|
ProcessorType, dataSources.Count);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
|
||||||
|
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
|
||||||
|
|
||||||
|
foreach (var baseCode in baseCodes)
|
||||||
|
{
|
||||||
|
var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
|
||||||
|
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"9{baseCode}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sum values from all sources for positions 1-32
|
||||||
|
for (var i = 1; i < 33; i++)
|
||||||
|
{
|
||||||
|
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||||
|
ProcessHelper.SetValue(processedRecord, i, totalValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
|
||||||
|
ProcessorType, baseCode, codeRecords.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
|
||||||
|
{
|
||||||
|
var calculatedRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
|
||||||
|
ProcessorType, DynamicCodes!.Count);
|
||||||
|
|
||||||
|
foreach (var dynamicCode in DynamicCodes!)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
|
||||||
|
ProcessorType, dynamicCode.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||||
|
if (!calc.IsFormulaCorrect())
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
|
||||||
|
ProcessorType, calc.Expression, dynamicCode.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calculatedRecord = calc.CalculateT3(baseRecords);
|
||||||
|
calculatedRecords.Add(calculatedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
|
||||||
|
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
|
||||||
|
ProcessorType, dynamicCode.Desc1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
|
||||||
|
ProcessorType, calculatedRecords.Count);
|
||||||
|
|
||||||
|
return calculatedRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,344 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using DiunaBI.Core.Services.Calculations;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaT3MultiSourceYearSummaryProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private List<Record>? Sources { get; set; }
|
||||||
|
private List<Record>? DynamicCodes { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3MultiSourceYearSummaryProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
ILogger<MorskaT3MultiSourceYearSummaryProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load sources
|
||||||
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||||
|
if (Sources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source records not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load dynamic codes
|
||||||
|
DynamicCodes = processWorker.Records
|
||||||
|
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||||
|
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
|
||||||
|
ProcessorType, Year, Sources.Count, DynamicCodes.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources",
|
||||||
|
ProcessorType, Year, Sources!.Count);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records (sum by base codes with validation)
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Process dynamic codes if configured
|
||||||
|
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||||
|
{
|
||||||
|
var calculatedRecords = ProcessDynamicCodes(newRecords);
|
||||||
|
newRecords.AddRange(calculatedRecords);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = Sources!.Select(source =>
|
||||||
|
_db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault())
|
||||||
|
.OfType<Layer>()
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for year {Year}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
|
||||||
|
ProcessorType, dataSources.Count, Year);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
|
||||||
|
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
|
||||||
|
|
||||||
|
foreach (var baseCode in baseCodes)
|
||||||
|
{
|
||||||
|
var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
|
||||||
|
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"9{baseCode}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Validation record for double-checking calculations
|
||||||
|
var validationRecord = new Record();
|
||||||
|
|
||||||
|
// Sum values from all sources for positions 1-32 with validation
|
||||||
|
for (var i = 1; i < 33; i++)
|
||||||
|
{
|
||||||
|
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||||
|
ProcessHelper.SetValue(processedRecord, i, totalValue);
|
||||||
|
|
||||||
|
// Validation calculation (identical to main calculation)
|
||||||
|
var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||||
|
ProcessHelper.SetValue(validationRecord, i, validationValue);
|
||||||
|
|
||||||
|
// Validate that both calculations match
|
||||||
|
var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!);
|
||||||
|
if (difference > 0.01)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
|
||||||
|
ProcessorType, baseCode, codeRecords.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
|
||||||
|
{
|
||||||
|
var calculatedRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
|
||||||
|
ProcessorType, DynamicCodes!.Count);
|
||||||
|
|
||||||
|
foreach (var dynamicCode in DynamicCodes!)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
|
||||||
|
ProcessorType, dynamicCode.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||||
|
if (!calc.IsFormulaCorrect())
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
|
||||||
|
ProcessorType, calc.Expression, dynamicCode.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var calculatedRecord = calc.CalculateT3(baseRecords);
|
||||||
|
calculatedRecords.Add(calculatedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
|
||||||
|
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
|
||||||
|
ProcessorType, dynamicCode.Desc1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
|
||||||
|
ProcessorType, calculatedRecords.Count);
|
||||||
|
|
||||||
|
return calculatedRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,340 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3SingleSourceProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T3.SingleSource";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT3SingleSourceProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private int Month { get; set; }
|
||||||
|
private string? SourceLayerName { get; set; }
|
||||||
|
private string? Source { get; set; }
|
||||||
|
private Layer? SourceImportWorker { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3SingleSourceProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT3SingleSourceProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year and month
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||||
|
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Month record not found or invalid");
|
||||||
|
}
|
||||||
|
Month = month;
|
||||||
|
|
||||||
|
// Load source layer name
|
||||||
|
SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayerName))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("SourceLayer record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load source name
|
||||||
|
Source = GetRecordValue(processWorker.Records, "Source");
|
||||||
|
if (string.IsNullOrEmpty(Source))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, SourceLayerName, Source);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required");
|
||||||
|
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||||
|
|
||||||
|
// Find source import worker
|
||||||
|
SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled);
|
||||||
|
if (SourceImportWorker == null)
|
||||||
|
{
|
||||||
|
errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, Source);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderBy(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
|
||||||
|
ProcessorType, dataSources.Count);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records",
|
||||||
|
ProcessorType, allRecords.Count);
|
||||||
|
|
||||||
|
foreach (var baseRecord in dataSources.Last().Records!)
|
||||||
|
{
|
||||||
|
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||||
|
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = baseRecord.Code,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
ProcessDailyValues(processedRecord, codeRecords);
|
||||||
|
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ProcessDailyValues(Record processedRecord, List<Record> codeRecords)
|
||||||
|
{
|
||||||
|
var lastDayInMonth = DateTime.DaysInMonth(Year, Month);
|
||||||
|
|
||||||
|
// Day 1 - first value for the month
|
||||||
|
var firstVal = codeRecords
|
||||||
|
.Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1))
|
||||||
|
.MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
|
||||||
|
|
||||||
|
ProcessHelper.SetValue(processedRecord, 1, firstVal);
|
||||||
|
var previousValue = firstVal;
|
||||||
|
|
||||||
|
// Days 2 to last-1 - daily differences
|
||||||
|
for (var i = 2; i < lastDayInMonth; i++)
|
||||||
|
{
|
||||||
|
var dayVal = codeRecords
|
||||||
|
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month)
|
||||||
|
.MaxBy(x => x.CreatedAt)?.Value1;
|
||||||
|
|
||||||
|
if (dayVal == null)
|
||||||
|
{
|
||||||
|
ProcessHelper.SetValue(processedRecord, i, 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var processedVal = dayVal - previousValue;
|
||||||
|
ProcessHelper.SetValue(processedRecord, i, processedVal);
|
||||||
|
previousValue = (double)dayVal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Last day - special handling
|
||||||
|
var lastVal = codeRecords
|
||||||
|
.Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth))
|
||||||
|
.MaxBy(x => x.CreatedAt)?.Value1;
|
||||||
|
|
||||||
|
if (lastVal == null)
|
||||||
|
{
|
||||||
|
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy last value to position 32
|
||||||
|
var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
|
||||||
|
ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}",
|
||||||
|
ProcessorType, processedRecord.Code, valueToCopy);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (isNew)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT3SourceYearSummaryProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T3.SourceYearSummary";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT3SourceYearSummaryProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private string? Source { get; set; }
|
||||||
|
|
||||||
|
public MorskaT3SourceYearSummaryProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT3SourceYearSummaryProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load source
|
||||||
|
Source = GetRecordValue(processWorker.Records, "Source");
|
||||||
|
if (string.IsNullOrEmpty(Source))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Source);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Source);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources for all months
|
||||||
|
var dataSources = GetDataSources();
|
||||||
|
|
||||||
|
// Process records (sum all monthly values)
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources()
|
||||||
|
{
|
||||||
|
var dataSources = new List<Layer>();
|
||||||
|
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
var dataSource = _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSource != null)
|
||||||
|
{
|
||||||
|
dataSources.Add(dataSource);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
|
||||||
|
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
|
||||||
|
ProcessorType, month);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}",
|
||||||
|
ProcessorType, dataSources.Count, Year, Source);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
var baseRecords = dataSources.Last().Records!;
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
|
||||||
|
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
|
||||||
|
|
||||||
|
foreach (var baseRecord in baseRecords)
|
||||||
|
{
|
||||||
|
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||||
|
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = baseRecord.Code,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sum values from all months for positions 1-32
|
||||||
|
for (var position = 1; position <= 32; position++)
|
||||||
|
{
|
||||||
|
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position));
|
||||||
|
ProcessHelper.SetValue(processedRecord, position, totalValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
|
||||||
|
ProcessorType, baseRecord.Code, codeRecords.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,600 @@
|
|||||||
|
using System.Globalization;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Google.Apis.Sheets.v4.Data;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using DiunaBI.Core.Services;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT4R2Processor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T4.R2";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT4R2Processor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private List<Record>? Sources { get; set; }
|
||||||
|
private string? LayerName { get; set; }
|
||||||
|
private string? ReportSheetName { get; set; }
|
||||||
|
private string? InvoicesSheetName { get; set; }
|
||||||
|
|
||||||
|
public MorskaT4R2Processor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT4R2Processor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load sources
|
||||||
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||||
|
if (Sources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source records not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load layer name
|
||||||
|
LayerName = GetRecordValue(processWorker.Records, "LayerName");
|
||||||
|
if (string.IsNullOrEmpty(LayerName))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("LayerName record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load report sheet name
|
||||||
|
ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
|
||||||
|
if (string.IsNullOrEmpty(ReportSheetName))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("GoogleSheetName record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load invoices sheet name
|
||||||
|
InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices");
|
||||||
|
if (string.IsNullOrEmpty(InvoicesSheetName))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("GoogleSheetName-Invoices record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}",
|
||||||
|
ProcessorType, Year, Sources.Count, LayerName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||||
|
if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required");
|
||||||
|
if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required");
|
||||||
|
if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
|
||||||
|
ProcessorType, Year, Sources!.Count);
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Process records for all sources
|
||||||
|
var newRecords = ProcessSources();
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
// Update Google Sheets reports
|
||||||
|
UpdateReport(processedLayer.Id);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-{LayerName}";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessSources()
|
||||||
|
{
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
foreach (var source in Sources!)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing source {Source}",
|
||||||
|
ProcessorType, source.Desc1);
|
||||||
|
|
||||||
|
var sourceCodes = GetSourceCodes(source);
|
||||||
|
var sourceRecords = ProcessSourceData(source, sourceCodes);
|
||||||
|
newRecords.AddRange(sourceRecords);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records",
|
||||||
|
ProcessorType, source.Desc1, sourceRecords.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<int> GetSourceCodes(Record source)
|
||||||
|
{
|
||||||
|
var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}");
|
||||||
|
if (string.IsNullOrEmpty(rawSourceCodes))
|
||||||
|
{
|
||||||
|
return new List<int>();
|
||||||
|
}
|
||||||
|
|
||||||
|
return ProcessHelper.ParseCodes(rawSourceCodes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessSourceData(Record source, List<int> sourceCodes)
|
||||||
|
{
|
||||||
|
var sourceRecords = new List<Record>();
|
||||||
|
var lastSourceCodes = new List<string>();
|
||||||
|
|
||||||
|
// Process monthly data (1-12)
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes);
|
||||||
|
sourceRecords.AddRange(monthRecords);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process year summary (month 13)
|
||||||
|
var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes);
|
||||||
|
sourceRecords.AddRange(yearSummaryRecords);
|
||||||
|
|
||||||
|
return sourceRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessMonthData(Record source, List<int> sourceCodes, int month, List<string> lastSourceCodes)
|
||||||
|
{
|
||||||
|
var monthRecords = new List<Record>();
|
||||||
|
|
||||||
|
if (IsDataAvailableForMonth(month))
|
||||||
|
{
|
||||||
|
var dataSource = GetMonthDataSource(source, month);
|
||||||
|
if (dataSource != null)
|
||||||
|
{
|
||||||
|
lastSourceCodes.Clear();
|
||||||
|
lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!));
|
||||||
|
|
||||||
|
var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes);
|
||||||
|
var processedRecords = CreateMonthRecords(filteredRecords, source, month);
|
||||||
|
monthRecords.AddRange(processedRecords);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records",
|
||||||
|
ProcessorType, month, source.Desc1, processedRecords.Count);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Data source {DataSource} not found",
|
||||||
|
ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Future months - create zero value records (except for FK2)
|
||||||
|
if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0)
|
||||||
|
{
|
||||||
|
var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month);
|
||||||
|
monthRecords.AddRange(futureRecords);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}",
|
||||||
|
ProcessorType, futureRecords.Count, month);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return monthRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsDataAvailableForMonth(int month)
|
||||||
|
{
|
||||||
|
return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer? GetMonthDataSource(Record source, int month)
|
||||||
|
{
|
||||||
|
return _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> FilterRecords(IEnumerable<Record> records, List<int> sourceCodes)
|
||||||
|
{
|
||||||
|
return records
|
||||||
|
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> CreateMonthRecords(List<Record> filteredRecords, Record source, int month)
|
||||||
|
{
|
||||||
|
return filteredRecords.Select(x => new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"{x.Code}{month:D2}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow,
|
||||||
|
Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1,
|
||||||
|
Desc1 = x.Desc1
|
||||||
|
}).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> CreateFutureMonthRecords(List<string> lastSourceCodes, List<int> sourceCodes, int month)
|
||||||
|
{
|
||||||
|
return lastSourceCodes
|
||||||
|
.Where(x => sourceCodes.Contains(int.Parse(x)))
|
||||||
|
.Select(x => new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"{x}{month:D2}",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow,
|
||||||
|
Value1 = 0
|
||||||
|
}).ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessYearSummaryData(Record source, List<int> sourceCodes)
|
||||||
|
{
|
||||||
|
var dataSourceSum = _db.Layers
|
||||||
|
.Where(x => x.Type == LayerType.Processed &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled &&
|
||||||
|
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T"))
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSourceSum == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found",
|
||||||
|
ProcessorType, $"{Year}/13-{source.Desc1}-T3");
|
||||||
|
return new List<Record>();
|
||||||
|
}
|
||||||
|
|
||||||
|
var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes);
|
||||||
|
var yearSummaryRecords = filteredRecords.Select(x => new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = $"{x.Code}13",
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow,
|
||||||
|
Value1 = x.Value32
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}",
|
||||||
|
ProcessorType, yearSummaryRecords.Count, source.Desc1);
|
||||||
|
|
||||||
|
return yearSummaryRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateReport(Guid sourceId)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}",
|
||||||
|
ProcessorType, sourceId);
|
||||||
|
|
||||||
|
const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ";
|
||||||
|
|
||||||
|
var processedLayer = GetProcessedLayerData(sourceId);
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
var codesRow = GetCodesFromSheet(sheetId);
|
||||||
|
|
||||||
|
UpdateMonthlyData(sheetId, processedLayer, codesRow);
|
||||||
|
UpdateYearSummary(sheetId, processedLayer, codesRow);
|
||||||
|
UpdateTimestamps(sheetId, processedLayer);
|
||||||
|
UpdateInvoicesData(sheetId, processedLayer);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports",
|
||||||
|
ProcessorType);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}",
|
||||||
|
ProcessorType, sourceId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer? GetProcessedLayerData(Guid sourceId)
|
||||||
|
{
|
||||||
|
return _db.Layers
|
||||||
|
.Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
private IList<object> GetCodesFromSheet(string sheetId)
|
||||||
|
{
|
||||||
|
var request = _googleSheetValues.Get(sheetId, "C4:Z4");
|
||||||
|
var response = request.Execute();
|
||||||
|
return response.Values[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList<object> codesRow)
|
||||||
|
{
|
||||||
|
const int startRow = 6;
|
||||||
|
|
||||||
|
for (var month = 1; month <= 12; month++)
|
||||||
|
{
|
||||||
|
var values = new List<object>();
|
||||||
|
var monthStr = month < 10 ? $"0{month}" : month.ToString();
|
||||||
|
|
||||||
|
foreach (string code in codesRow)
|
||||||
|
{
|
||||||
|
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}");
|
||||||
|
values.Add(record?.Value1?.ToString() ?? "0");
|
||||||
|
}
|
||||||
|
|
||||||
|
var valueRange = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>> { values }
|
||||||
|
};
|
||||||
|
|
||||||
|
var row = (startRow + month).ToString();
|
||||||
|
var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}");
|
||||||
|
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
update.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet",
|
||||||
|
ProcessorType, month);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateYearSummary(string sheetId, Layer processedLayer, IList<object> codesRow)
|
||||||
|
{
|
||||||
|
const int startRow = 6;
|
||||||
|
var valuesSum = new List<object>();
|
||||||
|
var emptyRow = new List<object>();
|
||||||
|
|
||||||
|
foreach (string code in codesRow)
|
||||||
|
{
|
||||||
|
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
||||||
|
emptyRow.Add("");
|
||||||
|
valuesSum.Add(record?.Value1?.ToString() ?? "0");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert empty row before sum
|
||||||
|
var rowEmpty = (startRow + 13).ToString();
|
||||||
|
var valueRangeEmpty = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>> { emptyRow }
|
||||||
|
};
|
||||||
|
var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}");
|
||||||
|
updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateEmpty.Execute();
|
||||||
|
|
||||||
|
// Update sum row
|
||||||
|
var rowSum = (startRow + 14).ToString();
|
||||||
|
var valueRangeSum = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>> { valuesSum }
|
||||||
|
};
|
||||||
|
var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}");
|
||||||
|
updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateSum.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateTimestamps(string sheetId, Layer processedLayer)
|
||||||
|
{
|
||||||
|
// Update UTC time
|
||||||
|
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
var valueRangeUtcTime = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>> { new List<object> { timeUtc } }
|
||||||
|
};
|
||||||
|
var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1");
|
||||||
|
updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateTimeUtc.Execute();
|
||||||
|
|
||||||
|
// Update Warsaw time
|
||||||
|
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||||
|
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||||
|
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
var valueRangeWarsawTime = new ValueRange
|
||||||
|
{
|
||||||
|
Values = new List<IList<object>> { new List<object> { timeWarsaw } }
|
||||||
|
};
|
||||||
|
var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2");
|
||||||
|
updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateTimeWarsaw.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||||
|
ProcessorType, timeUtc, timeWarsaw);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void UpdateInvoicesData(string sheetId, Layer processedLayer)
|
||||||
|
{
|
||||||
|
var invoices = processedLayer.Records!
|
||||||
|
.Where(x => x.Code!.Length == 12)
|
||||||
|
.OrderByDescending(x => x.Code)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var invoicesValues = new List<IList<object>>();
|
||||||
|
var cleanUpValues = new List<IList<object>>();
|
||||||
|
|
||||||
|
foreach (var invoice in invoices)
|
||||||
|
{
|
||||||
|
var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture)
|
||||||
|
.ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL"));
|
||||||
|
|
||||||
|
var invoiceRow = new List<object>
|
||||||
|
{
|
||||||
|
invoiceDate, "", invoice.Desc1!, invoice.Value1!
|
||||||
|
};
|
||||||
|
invoicesValues.Add(invoiceRow);
|
||||||
|
|
||||||
|
var cleanupRow = new List<object> { "", "", "", "" };
|
||||||
|
cleanUpValues.Add(cleanupRow);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear existing data
|
||||||
|
var cleanupValueRange = new ValueRange { Values = cleanUpValues };
|
||||||
|
var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
|
||||||
|
cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
cleanupInvoices.Execute();
|
||||||
|
|
||||||
|
// Update with new data
|
||||||
|
var invoicesValueRange = new ValueRange { Values = invoicesValues };
|
||||||
|
var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
|
||||||
|
updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||||
|
updateInvoices.Execute();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet",
|
||||||
|
ProcessorType, invoices.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Google.Apis.Sheets.v4;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT4SingleSourceProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T4.SingleSource";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaT4SingleSourceProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private int Month { get; set; }
|
||||||
|
private string? SourceLayer { get; set; }
|
||||||
|
private string? Source { get; set; }
|
||||||
|
|
||||||
|
public MorskaT4SingleSourceProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaT4SingleSourceProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load month
|
||||||
|
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||||
|
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Month record not found or invalid");
|
||||||
|
}
|
||||||
|
Month = month;
|
||||||
|
|
||||||
|
// Load source layer
|
||||||
|
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayer))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("SourceLayer record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load source
|
||||||
|
Source = GetRecordValue(processWorker.Records, "Source");
|
||||||
|
if (string.IsNullOrEmpty(Source))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, SourceLayer, Source);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
|
||||||
|
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, Source);
|
||||||
|
|
||||||
|
// Get source import worker
|
||||||
|
var sourceImportWorker = GetSourceImportWorker();
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data source
|
||||||
|
var dataSource = GetDataSource(sourceImportWorker);
|
||||||
|
|
||||||
|
// Process records (simple copy)
|
||||||
|
var newRecords = ProcessRecords(dataSource);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetSourceImportWorker()
|
||||||
|
{
|
||||||
|
var sourceImportWorker = _db.Layers
|
||||||
|
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (sourceImportWorker == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
|
||||||
|
|
||||||
|
return sourceImportWorker;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetDataSource(Layer sourceImportWorker)
|
||||||
|
{
|
||||||
|
var dataSource = _db.Layers
|
||||||
|
.Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (dataSource == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
|
||||||
|
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||||
|
|
||||||
|
return dataSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(Layer dataSource)
|
||||||
|
{
|
||||||
|
if (dataSource.Records == null || dataSource.Records.Count == 0)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType);
|
||||||
|
return new List<Record>();
|
||||||
|
}
|
||||||
|
|
||||||
|
var newRecords = dataSource.Records.Select(record => new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = record.Code,
|
||||||
|
Desc1 = record.Desc1,
|
||||||
|
Value1 = record.Value1,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,320 @@
|
|||||||
|
using DiunaBI.Core.Services;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|
||||||
|
public class MorskaT5LastValuesProcessor : MorskaBaseProcessor
|
||||||
|
{
|
||||||
|
public override string ProcessorType => "Morska.Process.T5.LastValues";
|
||||||
|
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaT5LastValuesProcessor> _logger;
|
||||||
|
|
||||||
|
// Configuration properties loaded from layer records
|
||||||
|
private int Year { get; set; }
|
||||||
|
private int Month { get; set; }
|
||||||
|
private string? SourceLayer { get; set; }
|
||||||
|
private string? Source { get; set; }
|
||||||
|
|
||||||
|
public MorskaT5LastValuesProcessor(
|
||||||
|
AppDbContext db,
|
||||||
|
ILogger<MorskaT5LastValuesProcessor> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void Process(Layer processWorker)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
|
||||||
|
// Load configuration from layer records
|
||||||
|
LoadConfiguration(processWorker);
|
||||||
|
|
||||||
|
// Validate required configuration
|
||||||
|
ValidateConfiguration();
|
||||||
|
|
||||||
|
// Perform the actual processing
|
||||||
|
PerformProcessing(processWorker);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||||
|
ProcessorType, processWorker.Name);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||||
|
ProcessorType, processWorker.Name, processWorker.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadConfiguration(Layer processWorker)
|
||||||
|
{
|
||||||
|
if (processWorker.Records == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("ProcessWorker has no records");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load year
|
||||||
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||||
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Year record not found or invalid");
|
||||||
|
}
|
||||||
|
Year = year;
|
||||||
|
|
||||||
|
// Load month
|
||||||
|
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||||
|
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Month record not found or invalid");
|
||||||
|
}
|
||||||
|
Month = month;
|
||||||
|
|
||||||
|
// Load source layer
|
||||||
|
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayer))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("SourceLayer record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load source
|
||||||
|
Source = GetRecordValue(processWorker.Records, "Source");
|
||||||
|
if (string.IsNullOrEmpty(Source))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException("Source record not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, SourceLayer, Source);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateConfiguration()
|
||||||
|
{
|
||||||
|
var errors = new List<string>();
|
||||||
|
|
||||||
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||||
|
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||||
|
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
|
||||||
|
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||||
|
|
||||||
|
if (errors.Any())
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void PerformProcessing(Layer processWorker)
|
||||||
|
{
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||||
|
ProcessorType, Year, Month, Source);
|
||||||
|
|
||||||
|
// Get source import worker
|
||||||
|
var sourceImportWorker = GetSourceImportWorker();
|
||||||
|
|
||||||
|
// Get or create processed layer
|
||||||
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||||
|
|
||||||
|
// Get data sources
|
||||||
|
var dataSources = GetDataSources(sourceImportWorker);
|
||||||
|
|
||||||
|
// Process records (get last values for each code)
|
||||||
|
var newRecords = ProcessRecords(dataSources);
|
||||||
|
|
||||||
|
// Save results
|
||||||
|
SaveProcessedLayer(processedLayer, newRecords);
|
||||||
|
|
||||||
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetSourceImportWorker()
|
||||||
|
{
|
||||||
|
var sourceImportWorker = _db.Layers
|
||||||
|
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (sourceImportWorker == null)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
|
||||||
|
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
|
||||||
|
|
||||||
|
return sourceImportWorker;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||||
|
{
|
||||||
|
var processedLayer = _db.Layers
|
||||||
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (processedLayer == null)
|
||||||
|
{
|
||||||
|
processedLayer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Type = LayerType.Processed,
|
||||||
|
ParentId = processWorker.Id,
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5";
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||||
|
ProcessorType, processedLayer.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedLayer;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Layer> GetDataSources(Layer sourceImportWorker)
|
||||||
|
{
|
||||||
|
var dataSources = _db.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.Where(x => x.ParentId == sourceImportWorker.Id &&
|
||||||
|
!x.IsDeleted && !x.IsCancelled)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
if (dataSources.Count == 0)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}",
|
||||||
|
ProcessorType, dataSources.Count, sourceImportWorker.Name);
|
||||||
|
|
||||||
|
return dataSources;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||||
|
{
|
||||||
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||||
|
var codes = allRecords.Select(x => x.Code).Distinct().ToList();
|
||||||
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records",
|
||||||
|
ProcessorType, codes.Count, allRecords.Count);
|
||||||
|
|
||||||
|
foreach (var code in codes)
|
||||||
|
{
|
||||||
|
var lastRecord = allRecords
|
||||||
|
.Where(x => x.Code == code)
|
||||||
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (lastRecord == null) continue;
|
||||||
|
|
||||||
|
var processedRecord = CreateProcessedRecord(lastRecord);
|
||||||
|
newRecords.Add(processedRecord);
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}",
|
||||||
|
ProcessorType, code, lastRecord.CreatedAt);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records",
|
||||||
|
ProcessorType, newRecords.Count);
|
||||||
|
|
||||||
|
return newRecords;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Record CreateProcessedRecord(Record lastRecord)
|
||||||
|
{
|
||||||
|
var processedRecord = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = lastRecord.Code,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
|
||||||
|
// Copy all values from positions 1-32
|
||||||
|
for (var i = 1; i < 33; i++)
|
||||||
|
{
|
||||||
|
var value = ProcessHelper.GetValue(lastRecord, i);
|
||||||
|
if (value != null)
|
||||||
|
{
|
||||||
|
ProcessHelper.SetValue(processedRecord, i, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return processedRecord;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||||
|
{
|
||||||
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||||
|
|
||||||
|
if (!existsInDb)
|
||||||
|
{
|
||||||
|
_db.Layers.Add(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_db.Layers.Update(processedLayer);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||||
|
}
|
||||||
|
|
||||||
|
SaveRecords(processedLayer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||||
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
// Remove existing records for this layer
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||||
|
ProcessorType, toDelete.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new records
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||||
|
ProcessorType, records.Count, layerId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||||
|
{
|
||||||
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,284 +0,0 @@
|
|||||||
using System.Globalization;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
using Google.Apis.Sheets.v4.Data;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
using DiunaBI.Core.Services.Calculations;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T1R1Processor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T1.R1";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T1R1Processor> _logger;
|
|
||||||
|
|
||||||
public T1R1Processor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T1R1Processor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
|
|
||||||
if (sources!.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-"))
|
|
||||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
for (var month = 1; month < 14; month++)
|
|
||||||
{
|
|
||||||
if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var records = new List<Record>();
|
|
||||||
foreach (var source in sources)
|
|
||||||
{
|
|
||||||
var monthCopy = month;
|
|
||||||
var dataSource = _db.Layers.Where(x =>
|
|
||||||
x.Type == LayerType.Processed &&
|
|
||||||
!x.IsDeleted && !x.IsCancelled &&
|
|
||||||
x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3")
|
|
||||||
).Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
if (dataSource == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found.");
|
|
||||||
}
|
|
||||||
|
|
||||||
var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault();
|
|
||||||
if (codesRecord != null)
|
|
||||||
{
|
|
||||||
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
|
|
||||||
records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!))));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
records.AddRange(dataSource.Records!);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dynamicCodes != null)
|
|
||||||
{
|
|
||||||
foreach (var dynamicCode in dynamicCodes)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (dynamicCode.Desc1 == null)
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
|
||||||
if (!calc.IsFormulaCorrect())
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
records.Add(calc.CalculateT1(records));
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newRecords.AddRange(records.Select(x => new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"{x.Code}{month:D2}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow,
|
|
||||||
Value1 = x.Value32
|
|
||||||
}
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
|
|
||||||
if (sheetName == null)
|
|
||||||
{
|
|
||||||
throw new Exception("GoogleSheetName record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
UpdateReport(processedLayer.Id, sheetName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateReport(Guid sourceId, string sheetName)
|
|
||||||
{
|
|
||||||
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
|
|
||||||
var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4");
|
|
||||||
var response = request.Execute();
|
|
||||||
|
|
||||||
var r1 = _db.Layers
|
|
||||||
.Where(x => x.Id == sourceId)
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var codesRow = response.Values[0];
|
|
||||||
|
|
||||||
var valueRange = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>>()
|
|
||||||
};
|
|
||||||
|
|
||||||
for (var i = 1; i <= 12; i++)
|
|
||||||
{
|
|
||||||
var values = new List<object>();
|
|
||||||
foreach (string code in codesRow)
|
|
||||||
{
|
|
||||||
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}");
|
|
||||||
if (record != null)
|
|
||||||
{
|
|
||||||
values.Add(record.Value1!.Value);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
values.Add("0");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
valueRange.Values.Add(values);
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum
|
|
||||||
var valuesSum = new List<object>();
|
|
||||||
var emptyRow = new List<object>();
|
|
||||||
foreach (string code in codesRow)
|
|
||||||
{
|
|
||||||
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
|
||||||
emptyRow.Add("");
|
|
||||||
if (record != null)
|
|
||||||
{
|
|
||||||
valuesSum.Add(record.Value1!.Value);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
valuesSum.Add("0");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
valueRange.Values.Add(emptyRow);
|
|
||||||
valueRange.Values.Add(valuesSum);
|
|
||||||
|
|
||||||
var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20");
|
|
||||||
update.ValueInputOption =
|
|
||||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
update.Execute();
|
|
||||||
|
|
||||||
// update time
|
|
||||||
var timeUtc = new List<object>
|
|
||||||
{
|
|
||||||
r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
|
||||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
|
||||||
var timeWarsaw = new List<object>
|
|
||||||
{
|
|
||||||
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var valueRangeTime = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>>()
|
|
||||||
};
|
|
||||||
valueRangeTime.Values.Add(timeUtc);
|
|
||||||
valueRangeTime.Values.Add(timeWarsaw);
|
|
||||||
|
|
||||||
var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
|
|
||||||
updateTimeUtc.ValueInputOption =
|
|
||||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateTimeUtc.Execute();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,227 +0,0 @@
|
|||||||
using System.Globalization;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
using Google.Apis.Sheets.v4.Data;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T1R3Processor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T1.R3";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T1R3Processor> _logger;
|
|
||||||
|
|
||||||
public T1R3Processor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T1R3Processor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1;
|
|
||||||
if (source == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$";
|
|
||||||
var dataSources = _db.Layers
|
|
||||||
.Where(x => !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.Include(layer => layer.Records!)
|
|
||||||
.AsNoTracking()
|
|
||||||
.AsEnumerable()
|
|
||||||
.Where(x => Regex.IsMatch(x.Name!, pattern))
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
foreach (var dataSource in dataSources)
|
|
||||||
{
|
|
||||||
var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!);
|
|
||||||
if (month == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"Month not found: {dataSource.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in dataSource.Records!)
|
|
||||||
{
|
|
||||||
if (record.Value1 == null) continue;
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
if (ProcessHelper.GetValue(record, i) == null) continue;
|
|
||||||
|
|
||||||
var newRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"{record.Code}{month}{i:D2}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow,
|
|
||||||
Value1 = i == 1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100,
|
|
||||||
Desc1 = record.Desc1
|
|
||||||
};
|
|
||||||
|
|
||||||
newRecords.Add(newRecord);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
UpdateReport(processedLayer.Id, year);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateReport(Guid sourceId, int year)
|
|
||||||
{
|
|
||||||
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";
|
|
||||||
|
|
||||||
var r3 = _db.Layers
|
|
||||||
.Where(x => x.Id == sourceId)
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
for (var i = 1; i <= 12; i++)
|
|
||||||
{
|
|
||||||
var sheetName = ProcessHelper.GetSheetName(i, year);
|
|
||||||
ValueRange? dataRangeResponse;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute();
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
continue; // Sheet not exist
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dataRangeResponse == null) continue; // Sheet not exist
|
|
||||||
var data = dataRangeResponse.Values;
|
|
||||||
|
|
||||||
var updateValueRange = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>>()
|
|
||||||
};
|
|
||||||
|
|
||||||
foreach (var row in data)
|
|
||||||
{
|
|
||||||
if (row.Count == 0) continue;
|
|
||||||
var code = row[0].ToString();
|
|
||||||
|
|
||||||
var updateRow = new List<object>();
|
|
||||||
|
|
||||||
for (var j = 1; j < 16; j++)
|
|
||||||
{
|
|
||||||
var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}");
|
|
||||||
if (codeRecord is { Value1: not null })
|
|
||||||
{
|
|
||||||
updateRow.Add(codeRecord.Value1);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
updateRow.Add("");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
updateValueRange.Values.Add(updateRow);
|
|
||||||
}
|
|
||||||
|
|
||||||
dataRangeResponse.Values = data;
|
|
||||||
var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200");
|
|
||||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
update.Execute();
|
|
||||||
|
|
||||||
// update time
|
|
||||||
var timeUtc = new List<object>
|
|
||||||
{
|
|
||||||
r3!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
|
||||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r3.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
|
||||||
var timeWarsaw = new List<object>
|
|
||||||
{
|
|
||||||
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var valueRangeTime = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>>()
|
|
||||||
};
|
|
||||||
valueRangeTime.Values.Add(timeUtc);
|
|
||||||
valueRangeTime.Values.Add(timeWarsaw);
|
|
||||||
|
|
||||||
var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
|
|
||||||
updateTimeUtc.ValueInputOption =
|
|
||||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateTimeUtc.Execute();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
using DiunaBI.Core.Services;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.MultiSourceCopySelectedCodes";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly ILogger<T3MultiSourceCopySelectedCodesProcessor> _logger;
|
|
||||||
|
|
||||||
public T3MultiSourceCopySelectedCodesProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
ILogger<T3MultiSourceCopySelectedCodesProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
|
|
||||||
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
|
|
||||||
if (sources!.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1;
|
|
||||||
if (codes == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Codes record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var codesList = ProcessHelper.ParseCodes(codes);
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AB-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var dataSources = sources.Select(source => _db.Layers
|
|
||||||
.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3"))
|
|
||||||
.Include(x => x.Records).AsNoTracking()
|
|
||||||
.FirstOrDefault())
|
|
||||||
.OfType<Layer>()
|
|
||||||
.ToList();
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("DataSources are empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
var newRecords = dataSources
|
|
||||||
.SelectMany(x => x.Records!)
|
|
||||||
.Where(x => codesList.Contains(int.Parse(x.Code!)))
|
|
||||||
.Select(x =>
|
|
||||||
{
|
|
||||||
var newRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = x.Code,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i));
|
|
||||||
}
|
|
||||||
return newRecord;
|
|
||||||
})
|
|
||||||
.ToList();
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
using DiunaBI.Core.Services;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
|
|
||||||
|
|
||||||
public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = new List<Layer>();
|
|
||||||
|
|
||||||
for (var i = 1; i < 13; i++)
|
|
||||||
{
|
|
||||||
var j = i;
|
|
||||||
var dataSource = _db.Layers.Where(x =>
|
|
||||||
x.Type == LayerType.Processed
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled
|
|
||||||
&& x.Name != null && x.Name.Contains($"{year}/{j:D2}-AB-T3"))
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
if (dataSource != null)
|
|
||||||
{
|
|
||||||
dataSources.Add(dataSource);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("DataSources are empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
|
||||||
|
|
||||||
foreach (var baseRecord in dataSources.Last().Records!)
|
|
||||||
{
|
|
||||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = baseRecord.Code,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i,
|
|
||||||
codeRecords.Sum(x => ProcessHelper.GetValue(x, i)));
|
|
||||||
}
|
|
||||||
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
}
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
using DiunaBI.Core.Services;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services.Calculations;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.MultiSourceSummary";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly ILogger<T3MultiSourceSummaryProcessor> _logger;
|
|
||||||
|
|
||||||
public T3MultiSourceSummaryProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
ILogger<T3MultiSourceSummaryProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
|
|
||||||
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
|
|
||||||
if (sources!.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AA-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3"))
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault())
|
|
||||||
.OfType<Layer>()
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("DataSources are empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
|
||||||
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
|
||||||
|
|
||||||
foreach (var baseCode in baseCodes)
|
|
||||||
{
|
|
||||||
var codeRecords = allRecords.Where(x =>
|
|
||||||
x.Code![1..] == baseCode)
|
|
||||||
.ToList();
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"9{baseCode}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i,
|
|
||||||
codeRecords.Sum(x => ProcessHelper.GetValue(x, i)));
|
|
||||||
}
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dynamic Codes
|
|
||||||
var dynamicCodes = processWorker.Records?
|
|
||||||
.Where(x => x.Code!.Contains("DynamicCode-"))
|
|
||||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
|
|
||||||
|
|
||||||
if (dynamicCodes != null && dynamicCodes.Count != 0)
|
|
||||||
{
|
|
||||||
foreach (var dynamicCode in dynamicCodes)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (dynamicCode.Desc1 == null)
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
|
||||||
if (!calc.IsFormulaCorrect())
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
newRecords.Add(calc.CalculateT3(newRecords));
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
_logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,198 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
using DiunaBI.Core.Services.Calculations;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.MultiSourceYearSummary";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger;
|
|
||||||
|
|
||||||
public T3MultiSourceYearSummaryProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
ILogger<T3MultiSourceYearSummaryProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
|
|
||||||
if (sources!.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3"))
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault())
|
|
||||||
.OfType<Layer>()
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("DataSources are empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
var allRecords = dataSources
|
|
||||||
.SelectMany(x => x.Records!).ToList();
|
|
||||||
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
|
||||||
|
|
||||||
foreach (var baseCode in baseCodes)
|
|
||||||
{
|
|
||||||
var codeRecords = allRecords.Where(x =>
|
|
||||||
x.Code![1..] == baseCode)
|
|
||||||
.ToList();
|
|
||||||
var codeRecordsValidation = allRecords.Where(x =>
|
|
||||||
x.Code![1..] == baseCode)
|
|
||||||
.ToList();
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"9{baseCode}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
var validationRecord = new Record();
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i,
|
|
||||||
codeRecords.Sum(x => ProcessHelper.GetValue(x, i)));
|
|
||||||
|
|
||||||
ProcessHelper.SetValue(validationRecord, i,
|
|
||||||
codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i)));
|
|
||||||
|
|
||||||
if (
|
|
||||||
double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) -
|
|
||||||
ProcessHelper.GetValue(validationRecord, i))!) > 0.01)
|
|
||||||
{
|
|
||||||
throw new Exception($"ValidationError: Code {baseCode}, " +
|
|
||||||
$"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " +
|
|
||||||
$"{ProcessHelper.GetValue(validationRecord, i)})");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dynamic Codes
|
|
||||||
var dynamicCodes = processWorker.Records?
|
|
||||||
.Where(x => x.Code!.Contains("DynamicCode-"))
|
|
||||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
|
|
||||||
|
|
||||||
if (dynamicCodes != null && dynamicCodes.Count != 0)
|
|
||||||
{
|
|
||||||
foreach (var dynamicCode in dynamicCodes)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (dynamicCode.Desc1 == null)
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
|
||||||
if (!calc.IsFormulaCorrect())
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
newRecords.Add(calc.CalculateT3(newRecords));
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
|
|
||||||
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
|
|
||||||
dynamicCode.Id, processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
_logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,177 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3SingleSourceProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.SingleSource";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T3SingleSourceProcessor> _logger;
|
|
||||||
|
|
||||||
public T3SingleSourceProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T3SingleSourceProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
|
|
||||||
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
|
|
||||||
if (sourceLayer == null)
|
|
||||||
{
|
|
||||||
throw new Exception("SourceLayer record not found");
|
|
||||||
}
|
|
||||||
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
|
|
||||||
if (sourceImportWorker == null)
|
|
||||||
{
|
|
||||||
throw new Exception("SourceImportWorkerL layer not found");
|
|
||||||
}
|
|
||||||
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
|
|
||||||
if (sourceLayer == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = _db.Layers
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.Where(x => x.ParentId == sourceImportWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderBy(x => x.CreatedAt)
|
|
||||||
.AsNoTracking()
|
|
||||||
.ToList();
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
|
||||||
|
|
||||||
foreach (var baseRecord in dataSources.Last().Records!)
|
|
||||||
{
|
|
||||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
|
||||||
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = baseRecord.Code,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
|
|
||||||
var lastDayInMonth = DateTime.DaysInMonth(year, month);
|
|
||||||
//day 1
|
|
||||||
var firstVal = codeRecords
|
|
||||||
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
|
|
||||||
ProcessHelper.SetValue(processedRecord, 1, firstVal);
|
|
||||||
var previousValue = firstVal;
|
|
||||||
//days 2-29/30
|
|
||||||
for (var i = 2; i < lastDayInMonth; i++)
|
|
||||||
{
|
|
||||||
var dayVal = codeRecords
|
|
||||||
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1;
|
|
||||||
if (dayVal == null)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i, 0);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
var processedVal = dayVal - previousValue;
|
|
||||||
ProcessHelper.SetValue(processedRecord, i, processedVal);
|
|
||||||
previousValue = (double)dayVal;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//last day
|
|
||||||
var lastVal = codeRecords
|
|
||||||
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1;
|
|
||||||
|
|
||||||
if (lastVal == null)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
// copy last value
|
|
||||||
var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
|
|
||||||
ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
|
|
||||||
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
}
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,147 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T3.SourceYearSummary";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T3SourceYearSummaryProcessor> _logger;
|
|
||||||
|
|
||||||
public T3SourceYearSummaryProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T3SourceYearSummaryProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
|
|
||||||
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
|
|
||||||
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
|
|
||||||
if (source == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = new List<Layer>();
|
|
||||||
for (var i = 1; i < 13; i++)
|
|
||||||
{
|
|
||||||
var j = i;
|
|
||||||
var dataSource = _db.Layers.Where(x =>
|
|
||||||
x.Type == LayerType.Processed
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled
|
|
||||||
&& x.Name != null && x.Name.Contains($"{year}/{j:D2}-{source}-T3"))
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
if (dataSource != null)
|
|
||||||
{
|
|
||||||
dataSources.Add(dataSource);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dataSources.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("DataSources are empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
|
||||||
|
|
||||||
foreach (var baseRecord in dataSources.Last().Records!)
|
|
||||||
{
|
|
||||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = baseRecord.Code,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i,
|
|
||||||
codeRecords.Sum(x => ProcessHelper.GetValue(x, i)));
|
|
||||||
}
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
_logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,386 +0,0 @@
|
|||||||
using System.Globalization;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
using Google.Apis.Sheets.v4.Data;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using DiunaBI.Core.Services;
|
|
||||||
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T4R2Processor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T4.R2";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T4R2Processor> _logger;
|
|
||||||
|
|
||||||
public T4R2Processor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T4R2Processor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
|
|
||||||
if (sources!.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1;
|
|
||||||
if (layerName == null)
|
|
||||||
{
|
|
||||||
throw new Exception("LayerName record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-{layerName}";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
foreach (var source in sources)
|
|
||||||
{
|
|
||||||
var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}")
|
|
||||||
?.Desc1;
|
|
||||||
var sourceCodes = new List<int>();
|
|
||||||
if (rawSourceCodes != null)
|
|
||||||
{
|
|
||||||
sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<string> lastSourceCodes = [];
|
|
||||||
|
|
||||||
for (var month = 1; month <= 12; month++)
|
|
||||||
{
|
|
||||||
if ((year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || year < DateTime.UtcNow.Year)
|
|
||||||
{
|
|
||||||
var monthCopy = month;
|
|
||||||
var dataSource = _db.Layers.Where(x =>
|
|
||||||
x.Type == LayerType.Processed &&
|
|
||||||
!x.IsDeleted && !x.IsCancelled &&
|
|
||||||
x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T")
|
|
||||||
)
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
if (dataSource != null)
|
|
||||||
{
|
|
||||||
lastSourceCodes = dataSource.Records!.Select(x => x.Code!).ToList();
|
|
||||||
var news = dataSource.Records!
|
|
||||||
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
|
|
||||||
.Select(x =>
|
|
||||||
{
|
|
||||||
var newRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"{x.Code}{month:D2}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow,
|
|
||||||
Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1,
|
|
||||||
Desc1 = x.Desc1
|
|
||||||
};
|
|
||||||
return newRecord;
|
|
||||||
}
|
|
||||||
).ToList();
|
|
||||||
newRecords.AddRange(news);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
|
|
||||||
$"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
//0 values for future months
|
|
||||||
if (source.Desc1 == "FK2" || lastSourceCodes.Count <= 0) continue;
|
|
||||||
var news = lastSourceCodes
|
|
||||||
.Where(x => sourceCodes.Contains(int.Parse(x)))
|
|
||||||
.Select(x =>
|
|
||||||
{
|
|
||||||
var newRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"{x}{month:D2}",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow,
|
|
||||||
Value1 = 0,
|
|
||||||
};
|
|
||||||
return newRecord;
|
|
||||||
}
|
|
||||||
).ToList();
|
|
||||||
newRecords.AddRange(news);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// year summary
|
|
||||||
var dataSourceSum = _db.Layers.Where(x =>
|
|
||||||
x.Type == LayerType.Processed &&
|
|
||||||
!x.IsDeleted && !x.IsCancelled &&
|
|
||||||
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
|
|
||||||
)
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
if (dataSourceSum != null)
|
|
||||||
{
|
|
||||||
var news = dataSourceSum.Records!
|
|
||||||
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
|
|
||||||
.Select(x =>
|
|
||||||
{
|
|
||||||
var newRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = $"{x.Code}13",
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow,
|
|
||||||
Value1 = x.Value32
|
|
||||||
};
|
|
||||||
return newRecord;
|
|
||||||
}
|
|
||||||
).ToList();
|
|
||||||
newRecords.AddRange(news);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
|
|
||||||
$"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
|
|
||||||
if (reportSheetName == null)
|
|
||||||
{
|
|
||||||
throw new Exception("GoogleSheetName record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var invoicesSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName-Invoices")?.Desc1;
|
|
||||||
if (invoicesSheetName == null)
|
|
||||||
{
|
|
||||||
throw new Exception("GoogleSheetName-Invoices record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName);
|
|
||||||
|
|
||||||
_logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName)
|
|
||||||
{
|
|
||||||
const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ";
|
|
||||||
var request = _googleSheetValues.Get(sheetId, "C4:Z4");
|
|
||||||
var response = request.Execute();
|
|
||||||
|
|
||||||
var r2 = _db.Layers
|
|
||||||
.Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
const int startRow = 6;
|
|
||||||
|
|
||||||
var codesRow = response.Values[0];
|
|
||||||
for (var i = 1; i <= 12; i++)
|
|
||||||
{
|
|
||||||
var values = new List<object>();
|
|
||||||
var month = i < 10 ? $"0{i}" : i.ToString();
|
|
||||||
var row = (startRow + i).ToString();
|
|
||||||
foreach (string code in codesRow)
|
|
||||||
{
|
|
||||||
var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}{month}");
|
|
||||||
if (record != null)
|
|
||||||
{
|
|
||||||
values.Add(record.Value1!.Value);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
values.Add("0");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var valueRange = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>> { values }
|
|
||||||
};
|
|
||||||
var update = _googleSheetValues.Update(valueRange, sheetId, $"{reportSheetName}!C{row}:XZ{row}");
|
|
||||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
update.Execute();
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum
|
|
||||||
var valuesSum = new List<object>();
|
|
||||||
var emptyRow = new List<object>();
|
|
||||||
var rowEmpty = (startRow + 13).ToString();
|
|
||||||
var rowSum = (startRow + 14).ToString();
|
|
||||||
foreach (string code in codesRow)
|
|
||||||
{
|
|
||||||
var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
|
||||||
emptyRow.Add("");
|
|
||||||
if (record != null)
|
|
||||||
{
|
|
||||||
valuesSum.Add(record.Value1!.Value);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
valuesSum.Add("0");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// insert empty row before sum
|
|
||||||
var valueRangeEmpty = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>> { emptyRow }
|
|
||||||
};
|
|
||||||
var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{reportSheetName}!C{rowEmpty}:XZ{rowEmpty}");
|
|
||||||
updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateEmpty.Execute();
|
|
||||||
|
|
||||||
var valueRangeSum = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>> { valuesSum }
|
|
||||||
};
|
|
||||||
var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{reportSheetName}!C{rowSum}:XZ{rowSum}");
|
|
||||||
updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateSum.Execute();
|
|
||||||
|
|
||||||
// update time
|
|
||||||
var timeUtc = new List<object>
|
|
||||||
{
|
|
||||||
r2!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var valueRangeUtcTime = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>> { timeUtc }
|
|
||||||
};
|
|
||||||
var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{reportSheetName}!G1");
|
|
||||||
updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateTimeUtc.Execute();
|
|
||||||
|
|
||||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
|
||||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r2.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
|
||||||
var timeWarsaw = new List<object>
|
|
||||||
{
|
|
||||||
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
|
|
||||||
};
|
|
||||||
var valueRangeWarsawTime = new ValueRange
|
|
||||||
{
|
|
||||||
Values = new List<IList<object>> { timeWarsaw }
|
|
||||||
};
|
|
||||||
var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{reportSheetName}!G2");
|
|
||||||
updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateTimeWarsaw.Execute();
|
|
||||||
|
|
||||||
//invoices
|
|
||||||
|
|
||||||
var invoices = r2.Records!.Where(x => x.Code!.Length == 12)
|
|
||||||
.OrderByDescending(x => x.Code);
|
|
||||||
|
|
||||||
var invoicesValues = new List<IList<object>>();
|
|
||||||
var cleanUpValues = new List<IList<object>>();
|
|
||||||
foreach (var invoice in invoices)
|
|
||||||
{
|
|
||||||
var invoiceDate =
|
|
||||||
DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture)
|
|
||||||
.ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL"));
|
|
||||||
var invoiceRow = new List<object>
|
|
||||||
{
|
|
||||||
invoiceDate,
|
|
||||||
"",
|
|
||||||
invoice.Desc1!,
|
|
||||||
invoice.Value1!
|
|
||||||
};
|
|
||||||
invoicesValues.Add(invoiceRow);
|
|
||||||
|
|
||||||
var cleanupRow = new List<object>
|
|
||||||
{
|
|
||||||
"", "", "", ""
|
|
||||||
};
|
|
||||||
cleanUpValues.Add(cleanupRow);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
var cleanupValueRange = new ValueRange { Values = cleanUpValues };
|
|
||||||
var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{invoicesSheetName}!A6:E");
|
|
||||||
cleanupInvoices.ValueInputOption =
|
|
||||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
cleanupInvoices.Execute();
|
|
||||||
|
|
||||||
|
|
||||||
var invoicesValueRange = new ValueRange { Values = invoicesValues };
|
|
||||||
var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{invoicesSheetName}!A6:E");
|
|
||||||
updateInvoices.ValueInputOption =
|
|
||||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
|
||||||
updateInvoices.Execute();
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
using Google.Apis.Sheets.v4;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T4SingleSourceProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T4.SingleSource";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|
||||||
private readonly ILogger<T4SingleSourceProcessor> _logger;
|
|
||||||
|
|
||||||
public T4SingleSourceProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|
||||||
ILogger<T4SingleSourceProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_googleSheetValues = googleSheetValues;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
|
|
||||||
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
|
|
||||||
if (sourceLayer == null)
|
|
||||||
{
|
|
||||||
throw new Exception("SourceLayer record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
|
|
||||||
if (sourceImportWorker == null)
|
|
||||||
{
|
|
||||||
throw new Exception("SourceImportWorker layer not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
|
|
||||||
if (source == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Source record not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T4";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var dataSource = _db.Layers
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.Where(x => x.ParentId == sourceImportWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.AsNoTracking()
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
if (dataSource == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"DataSource not found, {sourceImportWorker.Name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
var newRecords = dataSource.Records!.Select(record => new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = record.Code,
|
|
||||||
Desc1 = record.Desc1,
|
|
||||||
Value1 = record.Value1,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
}).ToList();
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
{
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
|
|
||||||
_logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
||||||
processWorker.Name, processWorker.Id);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
using DiunaBI.Core.Services;
|
|
||||||
using Microsoft.EntityFrameworkCore;
|
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Database.Context;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
|
||||||
|
|
||||||
public class T5LastValuesProcessor : MorskaBaseProcessor
|
|
||||||
{
|
|
||||||
public override string ProcessorType => "T5.LastValues";
|
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
|
||||||
private readonly ILogger<T5LastValuesProcessor> _logger;
|
|
||||||
|
|
||||||
public T5LastValuesProcessor(
|
|
||||||
AppDbContext db,
|
|
||||||
ILogger<T5LastValuesProcessor> logger)
|
|
||||||
{
|
|
||||||
_db = db;
|
|
||||||
_logger = logger;
|
|
||||||
}
|
|
||||||
public override void Process(Layer processWorker)
|
|
||||||
{
|
|
||||||
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
|
|
||||||
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
|
|
||||||
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
|
|
||||||
if (sourceLayer == null) throw new Exception("SourceLayer record not found");
|
|
||||||
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
|
|
||||||
if (sourceImportWorker == null) throw new Exception("SourceImportWorker layer not found");
|
|
||||||
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
|
|
||||||
if (sourceLayer == null) throw new Exception("Source record not found");
|
|
||||||
|
|
||||||
var processedLayer = _db.Layers
|
|
||||||
.Where(x => x.ParentId == processWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.FirstOrDefault();
|
|
||||||
|
|
||||||
var isNew = false;
|
|
||||||
if (processedLayer == null)
|
|
||||||
{
|
|
||||||
isNew = true;
|
|
||||||
processedLayer = new Layer
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Type = LayerType.Processed,
|
|
||||||
ParentId = processWorker.Id,
|
|
||||||
Number = _db.Layers.Count() + 1
|
|
||||||
};
|
|
||||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T5";
|
|
||||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
}
|
|
||||||
|
|
||||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
|
||||||
|
|
||||||
var dataSources = _db.Layers
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.Where(x => x.ParentId == sourceImportWorker.Id
|
|
||||||
&& !x.IsDeleted && !x.IsCancelled)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.AsNoTracking()
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (dataSources.Count == 0) throw new Exception($"DataSource is empty, {sourceImportWorker.Name}");
|
|
||||||
|
|
||||||
var codes = dataSources.SelectMany(x => x.Records!).Select(x => x.Code).Distinct().ToList();
|
|
||||||
|
|
||||||
foreach (var code in codes)
|
|
||||||
{
|
|
||||||
var lastRecord = dataSources.SelectMany(x => x.Records!).Where(x => x.Code == code).OrderByDescending(x => x.CreatedAt).FirstOrDefault();
|
|
||||||
if (lastRecord == null) continue;
|
|
||||||
|
|
||||||
var processedRecord = new Record
|
|
||||||
{
|
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = code,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
|
|
||||||
for (var i = 1; i < 33; i++)
|
|
||||||
{
|
|
||||||
if (ProcessHelper.GetValue(lastRecord, i) != null)
|
|
||||||
{
|
|
||||||
ProcessHelper.SetValue(processedRecord, i, ProcessHelper.GetValue(lastRecord, i));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newRecords.Add(processedRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isNew)
|
|
||||||
_db.Layers.Add(processedLayer);
|
|
||||||
else
|
|
||||||
_db.Layers.Update(processedLayer);
|
|
||||||
SaveRecords(processedLayer.Id, newRecords);
|
|
||||||
_db.SaveChanges();
|
|
||||||
}
|
|
||||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
||||||
{
|
|
||||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
||||||
if (toDelete.Count > 0)
|
|
||||||
{
|
|
||||||
_db.Records.RemoveRange(toDelete);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
|
||||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.CreatedAt = DateTime.UtcNow;
|
|
||||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
||||||
record.ModifiedAt = DateTime.UtcNow;
|
|
||||||
record.LayerId = layerId;
|
|
||||||
_db.Records.Add(record);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,7 @@ using System.IdentityModel.Tokens.Jwt;
|
|||||||
using System.Security.Claims;
|
using System.Security.Claims;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using Microsoft.AspNetCore.Authorization;
|
|||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|||||||
@@ -5,9 +5,9 @@ using Microsoft.AspNetCore.Authorization;
|
|||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Cloud.Firestore;
|
using DiunaBI.Core.Interfaces;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|
||||||
@@ -238,51 +238,6 @@ public class LayersController : Controller
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[HttpGet]
|
|
||||||
[Route("AutoImportWithQueue/{apiKey}")]
|
|
||||||
[AllowAnonymous]
|
|
||||||
public IActionResult AutoImportWithQueue(string apiKey)
|
|
||||||
{
|
|
||||||
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
|
|
||||||
{
|
|
||||||
_logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey);
|
|
||||||
return Unauthorized();
|
|
||||||
}
|
|
||||||
|
|
||||||
var importWorkerLayers = _db.Layers
|
|
||||||
.Include(x => x.Records)
|
|
||||||
.Where(x =>
|
|
||||||
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
|
||||||
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
|
|
||||||
)
|
|
||||||
.OrderBy(x => x.CreatedAt)
|
|
||||||
.AsNoTracking()
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (importWorkerLayers.Count == 0)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("AutoImportQueue: No layers to import");
|
|
||||||
return Ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count);
|
|
||||||
|
|
||||||
foreach (var importWorker in importWorkerLayers)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
// Queue job implementation would go here
|
|
||||||
_logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id);
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
_logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})",
|
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
[HttpGet]
|
[HttpGet]
|
||||||
[Route("ProcessQueue/{apiKey}")]
|
[Route("ProcessQueue/{apiKey}")]
|
||||||
[AllowAnonymous]
|
[AllowAnonymous]
|
||||||
@@ -673,7 +628,9 @@ public class LayersController : Controller
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
[HttpGet]
|
||||||
|
[Route("CheckProcessors")]
|
||||||
|
[AllowAnonymous]
|
||||||
|
|
||||||
private static void WriteToConsole(params string[] messages)
|
private static void WriteToConsole(params string[] messages)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -21,7 +21,6 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
||||||
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
using Google.Apis.Auth.OAuth2;
|
|
||||||
using Google.Cloud.Firestore;
|
|
||||||
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.IdentityModel.Tokens;
|
using Microsoft.IdentityModel.Tokens;
|
||||||
using System.IdentityModel.Tokens.Jwt;
|
using System.IdentityModel.Tokens.Jwt;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
|
using DiunaBI.Core.Interfaces;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// ✅ DODAJ SERILOG CONFIGURATION
|
if (builder.Environment.IsProduction())
|
||||||
|
{
|
||||||
builder.Host.UseSerilog((context, configuration) =>
|
builder.Host.UseSerilog((context, configuration) =>
|
||||||
{
|
{
|
||||||
configuration
|
configuration
|
||||||
@@ -24,8 +24,10 @@ builder.Host.UseSerilog((context, configuration) =>
|
|||||||
.Enrich.WithEnvironmentName()
|
.Enrich.WithEnvironmentName()
|
||||||
.Enrich.WithMachineName();
|
.Enrich.WithMachineName();
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
|
var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
|
||||||
|
|
||||||
builder.Services.AddDbContext<AppDbContext>(x =>
|
builder.Services.AddDbContext<AppDbContext>(x =>
|
||||||
{
|
{
|
||||||
x.UseSqlServer(connectionString);
|
x.UseSqlServer(connectionString);
|
||||||
@@ -50,7 +52,6 @@ builder.Services.AddCors(options =>
|
|||||||
|
|
||||||
builder.Services.AddControllers();
|
builder.Services.AddControllers();
|
||||||
|
|
||||||
|
|
||||||
builder.Services.AddAuthentication(options =>
|
builder.Services.AddAuthentication(options =>
|
||||||
{
|
{
|
||||||
options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
|
options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
|
||||||
@@ -66,11 +67,10 @@ builder.Services.AddAuthentication(options =>
|
|||||||
ValidateIssuerSigningKey = true,
|
ValidateIssuerSigningKey = true,
|
||||||
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!))
|
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!))
|
||||||
};
|
};
|
||||||
|
|
||||||
});
|
});
|
||||||
builder.Services.AddAuthentication();
|
|
||||||
|
|
||||||
// Zarejestruj Google Sheets dependencies
|
// Google Sheets dependencies
|
||||||
|
Console.WriteLine("Adding Google Sheets dependencies...");
|
||||||
builder.Services.AddSingleton<GoogleSheetsHelper>();
|
builder.Services.AddSingleton<GoogleSheetsHelper>();
|
||||||
builder.Services.AddSingleton<GoogleDriveHelper>();
|
builder.Services.AddSingleton<GoogleDriveHelper>();
|
||||||
builder.Services.AddSingleton<SpreadsheetsResource.ValuesResource>(provider =>
|
builder.Services.AddSingleton<SpreadsheetsResource.ValuesResource>(provider =>
|
||||||
@@ -90,6 +90,8 @@ builder.Services.AddSingleton<PluginManager>();
|
|||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
|
|
||||||
|
if (app.Environment.IsProduction())
|
||||||
|
{
|
||||||
app.UseSerilogRequestLogging(options =>
|
app.UseSerilogRequestLogging(options =>
|
||||||
{
|
{
|
||||||
options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms";
|
options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms";
|
||||||
@@ -108,14 +110,25 @@ app.UseSerilogRequestLogging(options =>
|
|||||||
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
|
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plugin initialization
|
||||||
var pluginManager = app.Services.GetRequiredService<PluginManager>();
|
var pluginManager = app.Services.GetRequiredService<PluginManager>();
|
||||||
var executablePath = Assembly.GetExecutingAssembly().Location;
|
var executablePath = Assembly.GetExecutingAssembly().Location;
|
||||||
var executableDir = Path.GetDirectoryName(executablePath)!;
|
var executableDir = Path.GetDirectoryName(executablePath)!;
|
||||||
var pluginsPath = Path.Combine(executableDir, "Plugins");
|
var pluginsPath = Path.Combine(executableDir, "Plugins");
|
||||||
|
|
||||||
|
if (app.Environment.IsProduction())
|
||||||
|
{
|
||||||
Log.Information("Starting DiunaBI application");
|
Log.Information("Starting DiunaBI application");
|
||||||
Log.Information("Loading plugins from: {PluginsPath}", pluginsPath);
|
Log.Information("Loading plugins from: {PluginsPath}", pluginsPath);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var logger = app.Services.GetRequiredService<ILogger<Program>>();
|
||||||
|
logger.LogInformation("Starting DiunaBI application (Development)");
|
||||||
|
logger.LogInformation("Loading plugins from: {PluginsPath}", pluginsPath);
|
||||||
|
}
|
||||||
|
|
||||||
pluginManager.LoadPluginsFromDirectory(pluginsPath);
|
pluginManager.LoadPluginsFromDirectory(pluginsPath);
|
||||||
|
|
||||||
@@ -143,5 +156,7 @@ app.MapControllers();
|
|||||||
|
|
||||||
app.Run();
|
app.Run();
|
||||||
|
|
||||||
// ✅ DODAJ CLEANUP
|
if (app.Environment.IsProduction())
|
||||||
|
{
|
||||||
Log.CloseAndFlush();
|
Log.CloseAndFlush();
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 17
|
# Visual Studio Version 17
|
||||||
VisualStudioVersion = 17.0.31903.59
|
VisualStudioVersion = 17.0.31903.59
|
||||||
@@ -9,8 +9,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Core", "DiunaBI.Cor
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Database", "DiunaBI.Database\DiunaBI.Database.csproj", "{8C346BEA-A209-4E8F-A6BF-70B42D9106C8}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ export const environment = {
|
|||||||
appName: "LOCAL_DiunaBI",
|
appName: "LOCAL_DiunaBI",
|
||||||
production: false,
|
production: false,
|
||||||
api: {
|
api: {
|
||||||
//url: "http://localhost:5400/api"
|
url: "http://localhost:5400/api"
|
||||||
url: "https://diunabi-morska.bim-it.pl/api"
|
//url: "https://diunabi-morska.bim-it.pl/api"
|
||||||
},
|
},
|
||||||
google: {
|
google: {
|
||||||
clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com"
|
clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com"
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
###
|
###
|
||||||
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-
|
GET http://localhost:5400/api/Layers/RunQueueJobs/10763478CB738D4ecb2h76g803478CB738D4e
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
###
|
###
|
||||||
GET http://localhost:5400/api/Layers/AutoProcess/10763478CB738D4ecb2h76g803478CB738D4e
|
GET http://localhost:5400/api/Layers/CheckProcessors
|
||||||
Timeout: 500000
|
Timeout: 500000
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
###
|
###
|
||||||
GET http://localhost:5400/api/Admin/BackupDatabase/10763478CB738D4ecb2h76g803478CB738D4e
|
GET http://localhost:5400/api/Layers/AddPluginName
|
||||||
|
|||||||
Reference in New Issue
Block a user