mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New processor.

Browse Source
This commit is contained in:
Michał Zieliński
2023-10-30 16:39:13 +01:00
parent a0d3ae8d72
commit bcbce3adcb
7 changed files with 690 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Frontend/src/app/modules/layers/layer-detail/layer-detail.component.ts
View File

@@ -66,7 +66,7 @@ export class LayerDetailComponent implements OnInit {
createColumns() {
this.displayedColumns = ['code'];
for (let i = 1; i < 32; i++) {
for (let i = 1; i < 33; i++) {
for (const record of this.document.records) {
const propertyName = `value${i}` as keyof typeof record;
if (record[propertyName] !== null) {

19
Utils/SQL_HELPERS.sql
View File

@@ -1,6 +1,8 @@
-- REMOVE LAYER
--DELETE FROM [diunabi-morska].[dbo].[Records] WHERE [LayerId] = 'bb4cc1ae-a94e-424d-9aa3-08dbba1c92b1';
--DELETE FROM [diunabi-morska].[dbo].[Layers] WHERE [Id] = 'bb4cc1ae-a94e-424d-9aa3-08dbba1c92b1';
--DECLARE @LayerId UNIQUEIDENTIFIER;
--SET @LayerId = '92ee1732-3900-4a8e-bb78-9bdc00332dd3';
--DELETE FROM [diunabi-morska].[dbo].[Records] WHERE [LayerId] = (SELECT @LayerId);
--DELETE FROM [diunabi-morska].[dbo].[Layers] WHERE [Id] = (SELECT @LayerId);
-- ADD IMPORT LAYER
@@ -71,17 +73,17 @@
--
-- ADD PROCESS LAYER
--DECLARE @LayerId UNIQUEIDENTIFIER;
--SET @LayerId = '171d25ce-7dde-4c84-8062-3e18d62c49fc';
--SET @LayerId = 'c433afde-5ca9-422a-9434-7985839dd414';
--
--INSERT INTO [diunabi-morska].[dbo].[Layers]
--([Id], [Number], [Source], [Name], [CreatedAt], [ModifiedAt], [IsDeleted],
--[CreatedById], [ModifiedById], [Type])
--VALUES ((SELECT @LayerId), 189, 'Manual', 'L189-A-PW_M7-2023/11-202311291149',
--VALUES ((SELECT @LayerId), 197, 'Manual', 'L197-A-PW_K5-2023/11-202310301538',
--GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2);
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Source', 'L183-A-IW_M7-2023/11-202310291127',
--VALUES ((SELECT NEWID()), 'Source', 'L181-A-IW_K5-2023/11-202310291031',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
@@ -91,6 +93,11 @@
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'ProcessType', 'Deaggregation',
--VALUES ((SELECT NEWID()), 'ProcessType', 'T3-SingleSource',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'IsEnabled', 'True',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--

15
WebAPI/Controllers/LayersController.cs
View File

@@ -194,7 +194,8 @@ namespace WebAPI.Controllers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True"))
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
)
.ToList();
layersToProcess = new List<Layer>();
@@ -253,12 +254,16 @@ namespace WebAPI.Controllers
switch (processType)
{
case "Copy":
//CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
//cp.process(sourceLayer, processWorker?.Id);
CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
cp.process(sourceLayer, processWorker?.Id);
break;
case "Deaggregation":
//DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
//dp.process(sourceLayer, processWorker?.Id);
DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
dp.process(sourceLayer, processWorker?.Id);
break;
case "T3-SingleSource":
T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
processor.process(sourceLayer, processWorker?.Id);
break;
}
logsController.AddEntry(new LogEntry

333
WebAPI/Migrations/20231030142419_Record.Value32.Designer.cs generated Normal file
View File

@@ -0,0 +1,333 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using WebAPI;
#nullable disable
namespace WebAPI.Migrations
{
[DbContext(typeof(AppDbContext))]
[Migration("20231030142419_Record.Value32")]
partial class RecordValue32
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.12")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("CreatedById")
.HasColumnType("uniqueidentifier");
b.Property<bool>("IsDeleted")
.HasColumnType("bit");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<Guid>("ModifiedById")
.HasColumnType("uniqueidentifier");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<int>("Number")
.HasColumnType("int");
b.Property<Guid?>("ParentId")
.HasColumnType("uniqueidentifier");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<int>("Type")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("CreatedById");
b.HasIndex("ModifiedById");
b.HasIndex("ParentId");
b.ToTable("Layers");
});
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
{
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<Guid>("SourceId")
.HasColumnType("uniqueidentifier");
b.HasKey("LayerId", "SourceId");
b.HasIndex("SourceId");
b.ToTable("ProcessSources");
});
modelBuilder.Entity("WebAPI.Models.Record", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<string>("Code")
.IsRequired()
.HasColumnType("nvarchar(max)");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<Guid>("CreatedById")
.HasColumnType("uniqueidentifier");
b.Property<string>("Desc1")
.HasColumnType("nvarchar(max)");
b.Property<string>("Desc2")
.HasColumnType("nvarchar(max)");
b.Property<string>("Desc3")
.HasColumnType("nvarchar(max)");
b.Property<string>("Desc4")
.HasColumnType("nvarchar(max)");
b.Property<string>("Desc5")
.HasColumnType("nvarchar(max)");
b.Property<bool>("IsDeleted")
.HasColumnType("bit");
b.Property<Guid>("LayerId")
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("ModifiedAt")
.HasColumnType("datetime2");
b.Property<Guid>("ModifiedById")
.HasColumnType("uniqueidentifier");
b.Property<float?>("Value1")
.HasColumnType("real");
b.Property<float?>("Value10")
.HasColumnType("real");
b.Property<float?>("Value11")
.HasColumnType("real");
b.Property<float?>("Value12")
.HasColumnType("real");
b.Property<float?>("Value13")
.HasColumnType("real");
b.Property<float?>("Value14")
.HasColumnType("real");
b.Property<float?>("Value15")
.HasColumnType("real");
b.Property<float?>("Value16")
.HasColumnType("real");
b.Property<float?>("Value17")
.HasColumnType("real");
b.Property<float?>("Value18")
.HasColumnType("real");
b.Property<float?>("Value19")
.HasColumnType("real");
b.Property<float?>("Value2")
.HasColumnType("real");
b.Property<float?>("Value20")
.HasColumnType("real");
b.Property<float?>("Value21")
.HasColumnType("real");
b.Property<float?>("Value22")
.HasColumnType("real");
b.Property<float?>("Value23")
.HasColumnType("real");
b.Property<float?>("Value24")
.HasColumnType("real");
b.Property<float?>("Value25")
.HasColumnType("real");
b.Property<float?>("Value26")
.HasColumnType("real");
b.Property<float?>("Value27")
.HasColumnType("real");
b.Property<float?>("Value28")
.HasColumnType("real");
b.Property<float?>("Value29")
.HasColumnType("real");
b.Property<float?>("Value3")
.HasColumnType("real");
b.Property<float?>("Value30")
.HasColumnType("real");
b.Property<float?>("Value31")
.HasColumnType("real");
b.Property<float?>("Value32")
.HasColumnType("real");
b.Property<float?>("Value4")
.HasColumnType("real");
b.Property<float?>("Value5")
.HasColumnType("real");
b.Property<float?>("Value6")
.HasColumnType("real");
b.Property<float?>("Value7")
.HasColumnType("real");
b.Property<float?>("Value8")
.HasColumnType("real");
b.Property<float?>("Value9")
.HasColumnType("real");
b.HasKey("Id");
b.HasIndex("CreatedById");
b.HasIndex("LayerId");
b.HasIndex("ModifiedById");
b.ToTable("Records");
});
modelBuilder.Entity("WebAPI.Models.User", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("CreatedAt")
.HasColumnType("datetime2");
b.Property<string>("Email")
.HasColumnType("nvarchar(max)");
b.Property<string>("UserName")
.HasMaxLength(50)
.HasColumnType("nvarchar(50)");
b.HasKey("Id");
b.ToTable("Users");
});
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.HasOne("WebAPI.Models.User", "CreatedBy")
.WithMany()
.HasForeignKey("CreatedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.User", "ModifiedBy")
.WithMany()
.HasForeignKey("ModifiedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.Layer", "Parent")
.WithMany()
.HasForeignKey("ParentId");
b.Navigation("CreatedBy");
b.Navigation("ModifiedBy");
b.Navigation("Parent");
});
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
{
b.HasOne("WebAPI.Models.Layer", "Source")
.WithMany("Sources")
.HasForeignKey("SourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Source");
});
modelBuilder.Entity("WebAPI.Models.Record", b =>
{
b.HasOne("WebAPI.Models.User", "CreatedBy")
.WithMany()
.HasForeignKey("CreatedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.Layer", null)
.WithMany("Records")
.HasForeignKey("LayerId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("WebAPI.Models.User", "ModifiedBy")
.WithMany()
.HasForeignKey("ModifiedById")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("CreatedBy");
b.Navigation("ModifiedBy");
});
modelBuilder.Entity("WebAPI.Models.Layer", b =>
{
b.Navigation("Records");
b.Navigation("Sources");
});
#pragma warning restore 612, 618
}
}
}

28
WebAPI/Migrations/20231030142419_Record.Value32.cs Normal file
View File

@@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace WebAPI.Migrations
{
/// <inheritdoc />
public partial class RecordValue32 : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<float>(
name: "Value32",
table: "Records",
type: "real",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "Value32",
table: "Records");
}
}
}

5
WebAPI/Migrations/AppDbContextModelSnapshot.cs
View File

@@ -17,7 +17,7 @@ namespace WebAPI.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "7.0.11")
.HasAnnotation("ProductVersion", "7.0.12")
.HasAnnotation("Relational:MaxIdentifierLength", 128);
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
@@ -204,6 +204,9 @@ namespace WebAPI.Migrations
b.Property<float?>("Value31")
.HasColumnType("real");
b.Property<float?>("Value32")
.HasColumnType("real");
b.Property<float?>("Value4")
.HasColumnType("real");

301
WebAPI/dataProcessors/t3.SingleSource.processor.cs Normal file
View File

@@ -0,0 +1,301 @@
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class T3SingleSourceProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
public T3SingleSourceProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void process(Layer sourceImportWorker, Guid? parentId)
{
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
}
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
}
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
}
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
}
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == parentId)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = parentId,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
List<Layer> dataSources = db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted)
.OrderBy(x => x.CreatedAt)
.ToList();
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
}
/*
foreach (Layer source in dataSources)
{
response.Sources.Add(new ProcessSource
{
LayerId = response.Id,
SourceId = source.Id
});
}
*/
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.First()?.Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = baseRecord.Code,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
float previousValue = 0;
//day 1
float firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
setValue(processedRecord, 1, firstVal);
previousValue = firstVal;
//days 2-29/30
for (int i=2; i<lastDayInMonth; i++)
{
float? dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (dayVal == null)
{
//TODO: missing day value? Should I log it?
setValue(processedRecord, i, 0);
} else
{
float processedVal = (dayVal ?? 0) - previousValue;
setValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0;
}
}
//last day
float? lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (lastVal == null)
{
setValue(processedRecord, lastDayInMonth, 0);
} else
{
setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
}
// copy last value
float? valueToCopy = codeRecords
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
setValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord);
}
if (isNew)
{
db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
private void setValue(Record record, int number, float? value)
{
switch (number)
{
case 1:
record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
}
}