T3SourceSummary processor
This commit is contained in:
Michał Zieliński
@@ -42,19 +42,19 @@ namespace WebAPI.Controllers
|
||||
try
|
||||
{
|
||||
IQueryable<Layer> response = db.Layers.Where(x => !x.IsDeleted);
|
||||
if (name != null)
|
||||
if (name != null)
|
||||
{
|
||||
response = response.Where(x => x.Name.Contains(name));
|
||||
response = response.Where(x => x.Name.Contains(name));
|
||||
}
|
||||
if (type != null)
|
||||
if (type != null)
|
||||
{
|
||||
response = response.Where(x => x.Type == type);
|
||||
response = response.Where(x => x.Type == type);
|
||||
}
|
||||
|
||||
return Ok(response
|
||||
.OrderByDescending(x => x.Number)
|
||||
.Skip(start).Take(limit).ToList());
|
||||
|
||||
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
@@ -194,14 +194,28 @@ namespace WebAPI.Controllers
|
||||
.Include(x => x.Records)
|
||||
.Where(x =>
|
||||
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
|
||||
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
|
||||
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
|
||||
//&& x.Number == 215
|
||||
)
|
||||
.ToList();
|
||||
|
||||
layersToProcess = new List<Layer>();
|
||||
|
||||
|
||||
foreach (Layer processWorker in processWorkerLayers)
|
||||
{
|
||||
string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
|
||||
if (processType == null)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = processWorker?.Name + " ProcessType record not found",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
}
|
||||
string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
|
||||
if (sourceName == null)
|
||||
{
|
||||
@@ -214,10 +228,21 @@ namespace WebAPI.Controllers
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
} else
|
||||
}
|
||||
|
||||
if (processType == "T3-SourceSummary")
|
||||
{
|
||||
if (processWorker != null)
|
||||
{
|
||||
T3SourceSummaryProcessor processor = new T3SourceSummaryProcessor(db, googleSheetValues, this);
|
||||
processor.process(processWorker);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
Layer sourceLayer = db.Layers.Include(x => x.Records)
|
||||
.Single(x => x.Name == sourceName);
|
||||
.Single(x => x.Name == sourceName);
|
||||
if (sourceLayer == null)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
@@ -229,55 +254,44 @@ namespace WebAPI.Controllers
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
} else
|
||||
}
|
||||
|
||||
string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
|
||||
string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
|
||||
string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
switch (processType)
|
||||
{
|
||||
string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
|
||||
if (processType == null)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = processWorker?.Name + " ProcessType record not found",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
} else
|
||||
{
|
||||
switch (processType)
|
||||
{
|
||||
case "Copy":
|
||||
CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
|
||||
cp.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
case "Deaggregation":
|
||||
DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
|
||||
dp.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
case "T3-SingleSource":
|
||||
T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
|
||||
processor.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
}
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"Process Success, {sourceLayer.Name}",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.process,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
case "Copy":
|
||||
CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
|
||||
cp.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
case "Deaggregation":
|
||||
DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
|
||||
dp.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
case "T3-SingleSource":
|
||||
T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
|
||||
processor.process(sourceLayer, processWorker?.Id);
|
||||
break;
|
||||
}
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"Process Success, {sourceLayer.Name}",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.process,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
} else if (endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
// TODO: make isEnabled = false - layer is out of date
|
||||
var name = processWorker.Name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Ok();
|
||||
}
|
||||
catch (Exception e)
|
||||
@@ -391,9 +405,9 @@ namespace WebAPI.Controllers
|
||||
layer.Records.Add(record);
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
AddLayer(layer, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
||||
|
||||
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"Import Success, {importWorker.Name}",
|
||||
@@ -401,7 +415,7 @@ namespace WebAPI.Controllers
|
||||
LogType = LogType.import,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
internal Layer AddLayer(Layer input, Guid currentUserId)
|
||||
{
|
||||
@@ -418,11 +432,11 @@ namespace WebAPI.Controllers
|
||||
{
|
||||
try
|
||||
{
|
||||
List<Record> toDelete = db.Records.Where(x => x.LayerId == id).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
List<Record> toDelete = db.Records.Where(x => x.LayerId == id).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
foreach (Record record in records)
|
||||
{
|
||||
record.CreatedById = currentUserId;
|
||||
|
||||
@@ -123,7 +123,7 @@ namespace WebAPI.dataProcessors
|
||||
|
||||
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
|
||||
foreach (Record baseRecord in dataSources.First()?.Records!)
|
||||
foreach (Record baseRecord in dataSources.Last()?.Records!)
|
||||
{
|
||||
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
|
||||
|
||||
110
WebAPI/dataProcessors/t3.SourceSummary.processor.cs
Normal file
110
WebAPI/dataProcessors/t3.SourceSummary.processor.cs
Normal file
@@ -0,0 +1,110 @@
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using WebAPI.Controllers;
|
||||
using WebAPI.Models;
|
||||
|
||||
namespace WebAPI.dataProcessors
|
||||
{
|
||||
public class T3SourceSummaryProcessor
|
||||
{
|
||||
private AppDbContext db;
|
||||
private SpreadsheetsResource.ValuesResource googleSheetValues;
|
||||
private LayersController controller;
|
||||
|
||||
public T3SourceSummaryProcessor(
|
||||
AppDbContext _db,
|
||||
SpreadsheetsResource.ValuesResource _googleSheetValues,
|
||||
LayersController _controller)
|
||||
{
|
||||
db = _db;
|
||||
googleSheetValues = _googleSheetValues;
|
||||
controller = _controller;
|
||||
}
|
||||
|
||||
public void process(Layer processWorker)
|
||||
{
|
||||
string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
|
||||
int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0");
|
||||
if (year == 0)
|
||||
{
|
||||
throw new Exception($"Year is empty, {processWorker!.Name}");
|
||||
}
|
||||
|
||||
Layer? processedLayer = db.Layers
|
||||
.Where(x => x.ParentId == processWorker!.Id
|
||||
&& !x.IsDeleted)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
bool isNew = false;
|
||||
if (processedLayer == null)
|
||||
{
|
||||
isNew = true;
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = "",
|
||||
Type = LayerType.processed,
|
||||
ParentId = processWorker!.Id,
|
||||
Number = db.Layers.Count() + 1,
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3";
|
||||
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.CreatedAt = DateTime.UtcNow;
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
}
|
||||
processedLayer.Sources = new List<ProcessSource>();
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
List<Record> newRecords = new List<Record>();
|
||||
|
||||
List<Layer> sources = new List<Layer>();
|
||||
for (int i=1; i<13; i++)
|
||||
{
|
||||
Layer? source = db.Layers.Where(x =>
|
||||
x.Type == LayerType.processed
|
||||
&& !x.IsDeleted
|
||||
&& x.Name.Contains($"{year}/{i}-{sourceName}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.FirstOrDefault();
|
||||
if (source != null)
|
||||
{
|
||||
sources.Add(source);
|
||||
}
|
||||
}
|
||||
|
||||
if (sources.Count == 0)
|
||||
{
|
||||
throw new Exception($"DataSources are empty, {processWorker!.Name}");
|
||||
}
|
||||
|
||||
List<Record> allRecords = sources.SelectMany(x => x.Records!).ToList();
|
||||
|
||||
foreach (Record baseRecord in sources.Last()?.Records!)
|
||||
{
|
||||
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
Record processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = baseRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = codeRecords.Sum(x => x.Value32)
|
||||
};
|
||||
newRecords.Add(processedRecord);
|
||||
}
|
||||
|
||||
if (isNew)
|
||||
{
|
||||
db.Layers.Add(processedLayer);
|
||||
} else
|
||||
{
|
||||
db.Layers.Update(processedLayer);
|
||||
}
|
||||
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
||||
db.SaveChanges();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user