mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new processor: T3-MultiSourceCopySelectedCodes

Browse Source
This commit is contained in:
Michał Zieliński
2023-12-20 18:39:43 +01:00
parent d73d3ee345
commit be80c90657
4 changed files with 226 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
Utils/SQL_HELPERS.sql
View File

@@ -71,19 +71,34 @@
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--
-- ADD PROCESS LAYER
--ADD PROCESS LAYER
--DECLARE @LayerId UNIQUEIDENTIFIER;
--SET @LayerId = 'c433afde-5ca9-422a-9434-7985839dd414';
--
--INSERT INTO [diunabi-morska].[dbo].[Layers]
--([Id], [Number], [Source], [Name], [CreatedAt], [ModifiedAt], [IsDeleted],
--[CreatedById], [ModifiedById], [Type])
--VALUES ((SELECT @LayerId), 197, 'Manual', 'L197-A-PW_K5-2023/11-202310301538',
--VALUES ((SELECT @LayerId), 490, 'Manual', 'L197-A-PW_AB-2023/11-202311201714',
--GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2);
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Source', 'L181-A-IW_K5-2023/11-202310291031',
--VALUES ((SELECT NEWID()), 'Source', 'K5',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Source', 'M7',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Source', 'PU',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Source', 'FK',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
@@ -93,11 +108,27 @@
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'ProcessType', 'T3-SingleSource',
--VALUES ((SELECT NEWID()), 'ProcessType', 'T3-MultiSourceSummarySelectedCodes',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'IsEnabled', 'True',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Year', '2023',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Month', '11',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
--
--INSERT INTO [diunabi-morska].[dbo].[Records]
--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
--VALUES ((SELECT NEWID()), 'Codes', '501-503;505-505;581-595;601-603;605-605;681-695;701-703;705-705;781-795;801-803;805-805;881-895',
--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, '7dbb3e85-895d-4c23-b57d-aa8505a26d5f');
--

7
WebAPI/Controllers/LayersController.cs
View File

@@ -233,6 +233,7 @@ namespace WebAPI.Controllers
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") &&
x.Records!.Any(x => x.Code == "ProcessType" && x.Desc1 == type)
&& x.Number == 490
)
.OrderBy(x => x.CreatedAt)
.ToList();
@@ -362,6 +363,12 @@ namespace WebAPI.Controllers
processor.process(processWorker!);
break;
}
case "T3-MultiSourceCopySelectedCodes":
{
T3MultiSourceCopySelectedCodesProcessor processor = new T3MultiSourceCopySelectedCodesProcessor(db, googleSheetValues, this);
processor.process(processWorker!);
break;
}
}
logsController.AddEntry(new LogEntry
{

25
WebAPI/dataProcessors/ProcessHelper.cs
View File

@@ -108,7 +108,6 @@ namespace DiunaBIWebAPI.dataProcessors
break;
}
}
public static float? getValue(Record record, int number)
{
switch (number)
@@ -181,6 +180,30 @@ namespace DiunaBIWebAPI.dataProcessors
return null;
}
}
public static List<int> parseCodes(string codes)
{
List<int> codesList = new List<int>();
foreach (string code in codes.Split(';'))
{
string[] range = code.Split('-');
if (range.Length == 1)
{
codesList.Add(int.Parse(range[0]));
}
else if (range.Length == 2)
{
for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++)
{
codesList.Add(i);
}
}
else
{
throw new Exception($"Invalid code range: {code}");
}
}
return codesList;
}
}
}

160
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs Normal file
View File

@@ -0,0 +1,160 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class T3MultiSourceCopySelectedCodesProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
{
throw new Exception("Source record not found");
}
string? codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1;
if (codes == null)
{
throw new Exception("Codes record not found");
}
// create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512]
List<int> codesList = ProcessHelper.parseCodes(codes);
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Layer> dataSources = new List<Layer>();
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource);
}
}
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
}
List<Record> newRecords = dataSources
.SelectMany(x => x.Records!)
.Where(x => codesList.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
for (var i = 1; i < 33; i++) {
ProcessHelper.setValue(newRecord, i, ProcessHelper.getValue(x, i));
}
return newRecord;
})
.ToList();
/*
foreach (Record baseRecord in dataSources.Last()?.Records!)
{
List<Record> codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
.ToList();
List<Record> codeRecordsValidation = allRecords.Where(x =>
x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
.ToList();
Record processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = $"19{baseRecord.Code!.Substring(Math.Max(0, baseRecord.Code!.Length - 2))}",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
Record validationRecord = new Record();
for (var i = 1; i < 33; i++)
{
ProcessHelper.setValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
ProcessHelper.setValue(validationRecord, i,
codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i)));
if (
ProcessHelper.getValue(processedRecord,i) !=
ProcessHelper.getValue(validationRecord, i))
{
throw new Exception($"ValidationError: Code {baseRecord.Code!}, " +
$"Value{i} ({ProcessHelper.getValue(processedRecord, i)} | " +
$"{ProcessHelper.getValue(validationRecord, i)})");
}
}
newRecords.Add(processedRecord);
}
*/
if (isNew)
{
db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}
}