Deagregation processor
This commit is contained in:
Michał Zieliński
@@ -1,4 +1,7 @@
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System.Globalization;
|
||||
using WebAPI.Models;
|
||||
|
||||
@@ -7,21 +10,98 @@ namespace WebAPI.dataProcessors
|
||||
public class CopyProcessor
|
||||
{
|
||||
private AppDbContext db;
|
||||
private SpreadsheetsResource.ValuesResource googleSheetValues;
|
||||
|
||||
public CopyProcessor(
|
||||
AppDbContext _db)
|
||||
AppDbContext _db,
|
||||
SpreadsheetsResource.ValuesResource _googleSheetValues)
|
||||
{
|
||||
db = _db;
|
||||
googleSheetValues = _googleSheetValues;
|
||||
}
|
||||
|
||||
public Layer process(Layer sourceLayer)
|
||||
{
|
||||
Layer dataSource = this.db.Layers
|
||||
.Where(x => x.ParentId == sourceLayer.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.First();
|
||||
|
||||
return null;
|
||||
public Layer process(Layer sourceImportWorker, Guid? parentId)
|
||||
{
|
||||
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
|
||||
if (importYearCell == null)
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
|
||||
if (importMonthCell == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
|
||||
if (importNameCell == null)
|
||||
{
|
||||
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
|
||||
string? name = nameResponse.Values[0][0].ToString();
|
||||
if (name == null)
|
||||
{
|
||||
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
|
||||
string? year = yearResponse.Values[0][0].ToString();
|
||||
if (year == null)
|
||||
{
|
||||
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
|
||||
string? month = monthResponse.Values[0][0].ToString();
|
||||
if (month == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
|
||||
Layer response = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = "",
|
||||
Type = LayerType.processed,
|
||||
ParentId = parentId,
|
||||
Number = db.Layers.Count() + 1,
|
||||
Records = new List<Record>(),
|
||||
Sources = new List<ProcessSource>()
|
||||
};
|
||||
|
||||
Layer dataSource = db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.First();
|
||||
response.Sources.Add(new ProcessSource
|
||||
{
|
||||
LayerId = response.Id,
|
||||
SourceId = dataSource.Id
|
||||
});
|
||||
response.Name = $"L{response.Number}-P-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
|
||||
|
||||
foreach (Record source in dataSource.Records!)
|
||||
{
|
||||
Record record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = source.Code,
|
||||
Value1 = source.Value1,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
response.Records.Add(record);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,262 @@
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using WebAPI.Models;
|
||||
|
||||
namespace WebAPI.dataProcessors
|
||||
{
|
||||
public class deaggregationProcessor
|
||||
public class DeaggregationProcessor
|
||||
{
|
||||
private AppDbContext db;
|
||||
private AppDbContext db;
|
||||
private SpreadsheetsResource.ValuesResource googleSheetValues;
|
||||
|
||||
public deaggregationProcessor(
|
||||
AppDbContext _db)
|
||||
public DeaggregationProcessor(
|
||||
AppDbContext _db,
|
||||
SpreadsheetsResource.ValuesResource _googleSheetValues)
|
||||
{
|
||||
db = _db;
|
||||
googleSheetValues = _googleSheetValues;
|
||||
}
|
||||
|
||||
public Layer process()
|
||||
public Layer process(Layer sourceImportWorker, Guid? parentId)
|
||||
{
|
||||
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
|
||||
if (importYearCell == null)
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
|
||||
if (importMonthCell == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
|
||||
if (importNameCell == null)
|
||||
{
|
||||
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
|
||||
}
|
||||
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
|
||||
string? name = nameResponse.Values[0][0].ToString();
|
||||
if (name == null)
|
||||
{
|
||||
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
|
||||
int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
|
||||
if (year == null)
|
||||
{
|
||||
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
|
||||
string? month = monthResponse.Values[0][0].ToString();
|
||||
if (month == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
|
||||
}
|
||||
|
||||
Layer response = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = "",
|
||||
Type = LayerType.processed,
|
||||
ParentId = parentId,
|
||||
Number = db.Layers.Count() + 1,
|
||||
Records = new List<Record>(),
|
||||
// Sources = new List<ProcessSource>()
|
||||
};
|
||||
response.Name = $"L{response.Number}-P-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
|
||||
|
||||
List<Layer> dataSources = db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == sourceImportWorker.Id
|
||||
&& x.CreatedAt.Month == int.Parse(month)
|
||||
&& x.CreatedAt.Year == year
|
||||
&& !x.IsDeleted)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToList();
|
||||
|
||||
/*
|
||||
foreach (Layer source in dataSources)
|
||||
{
|
||||
response.Sources.Add(new ProcessSource
|
||||
{
|
||||
LayerId = response.Id,
|
||||
SourceId = source.Id
|
||||
});
|
||||
}
|
||||
*/
|
||||
|
||||
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
|
||||
|
||||
|
||||
foreach (Record baseRecord in dataSources.First()?.Records!)
|
||||
{
|
||||
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
|
||||
Record processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = baseRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
|
||||
float previousValue = 0;
|
||||
//day 1
|
||||
float firstVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault()?.Value1 ?? 0;
|
||||
setValue(processedRecord, 1, firstVal);
|
||||
previousValue = firstVal;
|
||||
//days 2-29/30
|
||||
|
||||
|
||||
for (int i=2; i<lastDayInMonth; i++)
|
||||
{
|
||||
float? dayVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault()?.Value1;
|
||||
if (dayVal == null)
|
||||
{
|
||||
setValue(processedRecord, i, 0);
|
||||
} else
|
||||
{
|
||||
float processedVal = (dayVal ?? 0) - previousValue;
|
||||
setValue(processedRecord, i, processedVal);
|
||||
previousValue = processedVal;
|
||||
}
|
||||
}
|
||||
//last day
|
||||
float? lastVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault()?.Value1;
|
||||
|
||||
if (lastVal == null)
|
||||
{
|
||||
setValue(processedRecord, lastDayInMonth, 0);
|
||||
} else
|
||||
{
|
||||
setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
|
||||
}
|
||||
|
||||
response.Records.Add(processedRecord);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
private void setValue(Record record, int number, float? value)
|
||||
{
|
||||
return null;
|
||||
switch (number)
|
||||
{
|
||||
case 1:
|
||||
record.Value1 = value;
|
||||
break;
|
||||
case 2:
|
||||
record.Value2 = value;
|
||||
break;
|
||||
case 3:
|
||||
record.Value3 = value;
|
||||
break;
|
||||
case 4:
|
||||
record.Value4 = value;
|
||||
break;
|
||||
case 5:
|
||||
record.Value5 = value;
|
||||
break;
|
||||
case 6:
|
||||
record.Value6 = value;
|
||||
break;
|
||||
case 7:
|
||||
record.Value7 = value;
|
||||
break;
|
||||
case 8:
|
||||
record.Value8 = value;
|
||||
break;
|
||||
case 9:
|
||||
record.Value9 = value;
|
||||
break;
|
||||
case 10:
|
||||
record.Value10 = value;
|
||||
break;
|
||||
case 11:
|
||||
record.Value11 = value;
|
||||
break;
|
||||
case 12:
|
||||
record.Value12 = value;
|
||||
break;
|
||||
case 13:
|
||||
record.Value13 = value;
|
||||
break;
|
||||
case 14:
|
||||
record.Value14 = value;
|
||||
break;
|
||||
case 15:
|
||||
record.Value15 = value;
|
||||
break;
|
||||
case 16:
|
||||
record.Value16 = value;
|
||||
break;
|
||||
case 17:
|
||||
record.Value17 = value;
|
||||
break;
|
||||
case 18:
|
||||
record.Value18 = value;
|
||||
break;
|
||||
case 19:
|
||||
record.Value19 = value;
|
||||
break;
|
||||
case 20:
|
||||
record.Value20 = value;
|
||||
break;
|
||||
case 21:
|
||||
record.Value21 = value;
|
||||
break;
|
||||
case 22:
|
||||
record.Value22 = value;
|
||||
break;
|
||||
case 23:
|
||||
record.Value23 = value;
|
||||
break;
|
||||
case 24:
|
||||
record.Value24 = value;
|
||||
break;
|
||||
case 25:
|
||||
record.Value25 = value;
|
||||
break;
|
||||
case 26:
|
||||
record.Value26 = value;
|
||||
break;
|
||||
case 27:
|
||||
record.Value27 = value;
|
||||
break;
|
||||
case 28:
|
||||
record.Value28 = value;
|
||||
break;
|
||||
case 29:
|
||||
record.Value29 = value;
|
||||
break;
|
||||
case 30:
|
||||
record.Value30 = value;
|
||||
break;
|
||||
case 31:
|
||||
record.Value31 = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user