mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AutoProcess Refactor. Logging improvement. MultiSourceSummaryProcessor

Browse Source
This commit is contained in:
Michał Zieliński
2023-11-06 22:52:07 +01:00
parent e7c413d9cf
commit 8c26698dc8
4 changed files with 297 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
WebAPI/Controllers/LayersController.cs
View File

@@ -186,20 +186,143 @@ namespace WebAPI.Controllers
return Unauthorized();
}
List<Layer> processWorkerLayers;
List<Layer> layersToProcess;
try
{
processWorkerLayers = db.Layers
List<Layer> processWorkerLayers = db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
//&& x.Number == 215
//&& x.Number == 262
)
.OrderBy(x => x.CreatedAt)
.ToList();
layersToProcess = new List<Layer>();
if (processWorkerLayers.Count() == 0)
{
logsController.AddEntry(new LogEntry
{
Title = "No Layers to process.",
Type = LogEntryType.info,
LogType = LogType.process,
CreatedAt = DateTime.UtcNow,
});
return Ok();
}
foreach (Layer processWorker in processWorkerLayers)
{
try
{
string name = processWorker.Name;
string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
if (year == null)
{
throw new Exception("Year record nod found");
}
if (int.Parse(year!) < DateTime.UtcNow.Year)
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = "processLayer is out of date. Should be disabled. Not processed.",
CreatedAt = DateTime.UtcNow
});
continue;
}
string? processType = processWorker?.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1;
if (processType == null)
{
throw new Exception("ProcessType record not found");
}
if (processType == "T3-SourceYearSummary")
{
if (int.Parse(year!) < DateTime.UtcNow.Year)
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = "processLayer is out of date. Should be disabled. Not processed",
CreatedAt = DateTime.UtcNow
});
continue;
}
T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this, logsController);
processor.process(processWorker!);
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
continue;
}
string? month = processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1;
if (month == null)
{
throw new Exception("Month record not found");
}
if (int.Parse(month!) < DateTime.UtcNow.Month)
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = "processLayer is out of date. Should be disabled.",
CreatedAt = DateTime.UtcNow
});
}
switch (processType!)
{
case "T3-SingleSource":
{
T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
processor.process(processWorker!);
break;
}
case "T3-MultiSourceSummary":
{
T3MultiSourceSummaryProcessor processor = new T3MultiSourceSummaryProcessor(db, googleSheetValues, this);
processor.process(processWorker!);
break;
}
}
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
} catch (Exception e)
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.error,
LogType = LogType.process,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
}
}
return Ok();
foreach (Layer processWorker in processWorkerLayers)
{
@@ -216,8 +339,8 @@ namespace WebAPI.Controllers
});
return BadRequest();
}
string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
if (sourceName == null)
List<Record>? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList();
if (sources != null && sources.Count() == 0)
{
logsController.AddEntry(new LogEntry
{
@@ -234,15 +357,17 @@ namespace WebAPI.Controllers
{
if (processWorker != null)
{
T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
processor.process(processWorker);
//T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
// processor.process(processWorker);
}
}
else
} else if (processType == "T3-MultiSourceSummary")
{
var tst = 5;
} else
{
Layer sourceLayer = db.Layers.Include(x => x.Records)
.Single(x => x.Name == sourceName);
.Single(x => x.Name == sources!.First().Desc1);
if (sourceLayer == null)
{
logsController.AddEntry(new LogEntry
@@ -262,21 +387,7 @@ namespace WebAPI.Controllers
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
switch (processType)
{
case "Copy":
CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
cp.process(sourceLayer, processWorker?.Id);
break;
case "Deaggregation":
DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
dp.process(sourceLayer, processWorker?.Id);
break;
case "T3-SingleSource":
T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
processor.process(sourceLayer, processWorker?.Id);
break;
}
logsController.AddEntry(new LogEntry
{
Title = $"Process Success, {sourceLayer.Name}",

116
WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs Normal file
View File

@@ -0,0 +1,116 @@
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class T3MultiSourceSummaryProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
public T3MultiSourceSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>();
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource);
}
}
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
}
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!)
{
List<Record> codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
.ToList();
Record processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = $"9{baseRecord.Code!.Substring(1)}",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = codeRecords.Sum(x => x.Value32)
};
newRecords.Add(processedRecord);
}
if (isNew)
{
db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}
}

81
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -24,55 +24,23 @@ namespace WebAPI.dataProcessors
controller = _controller;
}
public void process(Layer sourceImportWorker, Guid? parentId)
public void process(Layer processWorker)
{
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
}
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
}
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
}
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
}
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception("Source record not found");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == source);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == parentId)
.Where(x => x.ParentId == processWorker!.Id)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
@@ -85,10 +53,10 @@ namespace WebAPI.dataProcessors
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = parentId,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3";
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
@@ -98,6 +66,7 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
List<Layer> dataSources = db.Layers
@@ -110,16 +79,7 @@ namespace WebAPI.dataProcessors
{
throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
}
/*
foreach (Layer source in dataSources)
{
response.Sources.Add(new ProcessSource
{
LayerId = response.Id,
SourceId = source.Id
});
}
*/
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
@@ -135,11 +95,11 @@ namespace WebAPI.dataProcessors
ModifiedAt = DateTime.UtcNow
};
int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
int lastDayInMonth = DateTime.DaysInMonth(year, month);
float previousValue = 0;
//day 1
float firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
setValue(processedRecord, 1, firstVal);
@@ -148,7 +108,7 @@ namespace WebAPI.dataProcessors
for (int i=2; i<lastDayInMonth; i++)
{
float? dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (dayVal == null)
@@ -164,7 +124,7 @@ namespace WebAPI.dataProcessors
}
//last day
float? lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
@@ -194,6 +154,7 @@ namespace WebAPI.dataProcessors
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
private void setValue(Record record, int number, float? value)
{

41
WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
View File

@@ -7,29 +7,32 @@ namespace WebAPI.dataProcessors
{
public class T3SourceYearSummaryProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
public T3SourceYearSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
LayersController _controller,
LogsController _logsController
)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
logsController = _logsController;
}
public void process(Layer processWorker)
{
string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0");
if (year == 0)
string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception($"Year is empty, {processWorker!.Name}");
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
&& !x.IsDeleted)
@@ -48,7 +51,7 @@ namespace WebAPI.dataProcessors
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3";
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
@@ -60,29 +63,29 @@ namespace WebAPI.dataProcessors
List<Record> newRecords = new List<Record>();
List<Layer> sources = new List<Layer>();
List<Layer> dataSources = new List<Layer>();
for (int i=1; i<13; i++)
{
Layer? source = db.Layers.Where(x =>
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed
&& !x.IsDeleted
&& x.Name.Contains($"{year}/{i}-{sourceName}-T3"))
&& x.Name.Contains($"{year}/{i}-{source}-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (source != null)
if (dataSource != null)
{
sources.Add(source);
dataSources.Add(dataSource!);
}
}
if (sources.Count == 0)
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty, {processWorker!.Name}");
throw new Exception("DataSources are empty");
}
List<Record> allRecords = sources.SelectMany(x => x.Records!).ToList();
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in sources.Last()?.Records!)
foreach (Record baseRecord in dataSources.Last()?.Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record