mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AutoProcess Refactor. Logging improvement. MultiSourceSummaryProcessor

Browse Source
This commit is contained in:
Michał Zieliński
2023-11-06 22:52:07 +01:00
parent e7c413d9cf
commit 8c26698dc8
4 changed files with 297 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -24,55 +24,23 @@ namespace WebAPI.dataProcessors
controller = _controller;
}
public void process(Layer sourceImportWorker, Guid? parentId)
public void process(Layer processWorker)
{
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
}
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
}
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
}
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
}
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception("Source record not found");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == source);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == parentId)
.Where(x => x.ParentId == processWorker!.Id)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
@@ -85,10 +53,10 @@ namespace WebAPI.dataProcessors
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = parentId,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3";
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
@@ -98,6 +66,7 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
List<Layer> dataSources = db.Layers
@@ -110,16 +79,7 @@ namespace WebAPI.dataProcessors
{
throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
}
/*
foreach (Layer source in dataSources)
{
response.Sources.Add(new ProcessSource
{
LayerId = response.Id,
SourceId = source.Id
});
}
*/
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
@@ -135,11 +95,11 @@ namespace WebAPI.dataProcessors
ModifiedAt = DateTime.UtcNow
};
int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
int lastDayInMonth = DateTime.DaysInMonth(year, month);
float previousValue = 0;
//day 1
float firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
setValue(processedRecord, 1, firstVal);
@@ -148,7 +108,7 @@ namespace WebAPI.dataProcessors
for (int i=2; i<lastDayInMonth; i++)
{
float? dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (dayVal == null)
@@ -164,7 +124,7 @@ namespace WebAPI.dataProcessors
}
//last day
float? lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
@@ -194,6 +154,7 @@ namespace WebAPI.dataProcessors
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
private void setValue(Record record, int number, float? value)
{