mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AutoImport refactor

Browse Source
This commit is contained in:
Michał Zieliński
2023-11-09 15:17:22 +01:00
parent 96b710ad82
commit b07c52b945
8 changed files with 223 additions and 724 deletions
Download Patch File Download Diff File Expand all files Collapse all files

367
WebAPI/Controllers/LayersController.cs
View File

@@ -1,5 +1,6 @@
using System.Globalization;
using System.Xml.Serialization;
using DiunaBIWebAPI.dataImporters;
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@@ -66,9 +67,11 @@ namespace WebAPI.Controllers
{
try
{
Request.Headers.TryGetValue("userId", out var value);
Guid currentUserId = new Guid(value!);
return Ok(AddLayer(input, currentUserId).Id);
//Request.Headers.TryGetValue("userId", out var value);
//Guid currentUserId = new Guid(value!);
return Ok();
}
catch (Exception e)
{
@@ -92,31 +95,6 @@ namespace WebAPI.Controllers
}
}
[HttpGet]
[Route("parseGoogleSheet/{sheetId}")]
public IActionResult ParseGoogleSheet(string sheetId)
{
string sheetName = "KOSZTY";
Layer layer = new Layer();
layer.Source = "GoogleSheet";
layer.Number = db.Layers.Count() + 1;
var parser = new googleSheetParser(googleSheetValues);
dynamic parsedSheet = parser.parse(sheetId);
layer.Records = parsedSheet.records;
layer.Name = $"W{layer.Number}-I-{sheetName}-{parsedSheet.date}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
return Ok(layer);
}
[HttpPost]
[DisableRequestSizeLimit]
[Route("parseFile")]
public IActionResult ParseFile()
{
var parser = new csvParser();
return Ok(parser.parse(Request.Form.Files[0]));
}
[HttpGet]
[Route("exportToGoogleSheet/{id}")]
public IActionResult ExportToGoogleSheet(Guid id)
{
@@ -138,44 +116,97 @@ namespace WebAPI.Controllers
{
return Unauthorized();
}
List<Layer> importWorkerLayers;
List<Layer> layersToImport; ;
try
{
importWorkerLayers = db.Layers
.Include(x => x.Records)
.Where(x => x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker"))
.ToList();
layersToImport = new List<Layer>();
foreach (Layer layer in importWorkerLayers)
{
string startDate = layer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
string endDate = layer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
processImportWorker(layer);
}
}
}
catch (Exception e)
{
try
{
List<Layer> importWorkerLayers;
importWorkerLayers = db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker") &&
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
)
.OrderBy(x => x.CreatedAt)
.ToList();
if (importWorkerLayers.Count() == 0)
{
logsController.AddEntry(new LogEntry
{
Title = "No Layers to import.",
Type = LogEntryType.info,
LogType = LogType.import,
CreatedAt = DateTime.UtcNow,
});
return Ok();
}
foreach (Layer importWorker in importWorkerLayers)
{
try
{
string? startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
if (startDate == null)
{
throw new Exception("Year record nod found");
}
string? endDate = importWorker.Records!.Where(x => x.Code == "EndDate").First().Desc1;
if (endDate == null)
{
throw new Exception("Year record nod found");
}
var startDateParsed = DateTime.ParseExact(startDate!, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate!, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
MorskaImporter importer = new MorskaImporter(db, googleSheetValues, this);
importer.import(importWorker);
logsController.AddEntry(new LogEntry
{
Title = $"{importWorker!.Name}, {importWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{importWorker!.Name}, {importWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.import,
Message = "importLayer is out of date. Should be disabled. Not processed.",
CreatedAt = DateTime.UtcNow
});
}
} catch(Exception e)
{
logsController.AddEntry(new LogEntry
{
Title = $"{importWorker!.Name}, {importWorker.Id}",
Type = LogEntryType.error,
LogType = LogType.import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
}
}
return Ok();
} catch(Exception e)
{
logsController.AddEntry(new LogEntry
{
Title = "Import error",
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
return BadRequest(e.ToString());
return BadRequest(e.ToString());
}
return Ok();
}
[HttpGet]
[Route("AutoProcess/{apiKey}/{type}")]
[AllowAnonymous]
@@ -344,92 +375,6 @@ namespace WebAPI.Controllers
});
}
}
return Ok();
foreach (Layer processWorker in processWorkerLayers)
{
string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
if (processType == null)
{
logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.import,
Message = processWorker?.Name + " ProcessType record not found",
CreatedAt = DateTime.UtcNow
});
return BadRequest();
}
List<Record>? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList();
if (sources != null && sources.Count() == 0)
{
logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.import,
Message = processWorker?.Name + " Source record not found",
CreatedAt = DateTime.UtcNow
});
return BadRequest();
}
if (processType == "T3-SourceSummary")
{
if (processWorker != null)
{
//T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
// processor.process(processWorker);
}
} else if (processType == "T3-MultiSourceSummary")
{
var tst = 5;
} else
{
Layer sourceLayer = db.Layers.Include(x => x.Records)
.Single(x => x.Name == sources!.First().Desc1);
if (sourceLayer == null)
{
logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.import,
Message = processWorker?.Name + " Source layer not found " + sourceLayer,
CreatedAt = DateTime.UtcNow
});
return BadRequest();
}
string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
logsController.AddEntry(new LogEntry
{
Title = $"Process Success, {sourceLayer.Name}",
Type = LogEntryType.info,
LogType = LogType.process,
CreatedAt = DateTime.UtcNow
});
} else if (endDateParsed.Date >= DateTime.UtcNow.Date)
{
// TODO: make isEnabled = false - layer is out of date
var name = processWorker.Name;
}
}
}
return Ok();
}
catch (Exception e)
@@ -445,127 +390,6 @@ namespace WebAPI.Controllers
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("checkDates")]
public IActionResult checkDates()
{
var warsawTZ = TimeZoneInfo.FindSystemTimeZoneById("Singapore Standard Time");
DateTime date = DateTime.UtcNow;
return Ok(date);
}
//
private void processImportWorker(Layer importWorker)
{
// get and check settings
string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {importWorker.Name}");
}
string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (checkSumCell == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1;
if (dataRange == null)
{
throw new Exception($"DataRange not found, {importWorker.Name}");
}
// open excel and read data
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values?[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
string? year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
Layer layer = new Layer
{
Source = "GoogleSheet",
Number = db.Layers.Count() + 1,
ParentId = importWorker.Id
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
layer.Type = LayerType.import;
layer.Records = new List<Record>();
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (int i = 0; i < data[1].Count; i++)
{
float value;
if (
data[0][i].ToString()?.Length > 0 &&
float.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
{
Record record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Records.Add(record);
};
}
AddLayer(layer, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
logsController.AddEntry(new LogEntry
{
Title = $"Import Success, {importWorker.Name}",
Type = LogEntryType.info,
LogType = LogType.import,
CreatedAt = DateTime.UtcNow
});
}
internal Layer AddLayer(Layer input, Guid currentUserId)
{
input.CreatedById = currentUserId;
input.ModifiedById = currentUserId;
input.CreatedAt = DateTime.UtcNow;
input.ModifiedAt = DateTime.UtcNow;
db.Layers.Add(input);
SaveRecords(input.Id, input.Records!, currentUserId);
db.SaveChanges();
return input;
}
internal void SaveRecords(Guid id, ICollection<Record> records, Guid currentUserId)
{
try
@@ -590,20 +414,5 @@ namespace WebAPI.Controllers
throw;
}
}
private void SaveSources(ICollection<ProcessSource> sources)
{
try
{
foreach (ProcessSource processSource in sources)
{
Console.WriteLine($"{processSource.LayerId} {processSource.SourceId}");
db.ProcessSources.Add(processSource);
}
}
catch (Exception)
{
throw;
}
}
}
}

2
WebAPI/DiunaBI-WebAPI.csproj
View File

@@ -33,8 +33,10 @@
<ItemGroup>
<None Remove="dataProcessors\" />
<None Remove="dataImporters\" />
</ItemGroup>
<ItemGroup>
<Folder Include="dataProcessors\" />
<Folder Include="dataImporters\" />
</ItemGroup>
</Project>

124
WebAPI/dataImporters/morska.importer.cs Normal file
View File

@@ -0,0 +1,124 @@
using System;
using System.Globalization;
using Google.Apis.Sheets.v4;
using WebAPI;
using WebAPI.Controllers;
using WebAPI.Models;
namespace DiunaBIWebAPI.dataImporters
{
public class MorskaImporter
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public MorskaImporter(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void import(Layer importWorker)
{
string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {importWorker.Name}");
}
string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (checkSumCell == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1;
if (dataRange == null)
{
throw new Exception($"DataRange not found, {importWorker.Name}");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values?[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
string? year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
Layer layer = new Layer
{
Source = "GoogleSheet",
Number = db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
List<Record> newRecords = new List<Record>();
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (int i = 0; i < data[1].Count; i++)
{
float value;
if (
data[0][i].ToString()?.Length > 0 &&
float.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
{
Record record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
};
}
db.Layers.Add(layer);
controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}
}

141
WebAPI/dataProcessors/copy.processor.cs
View File

@@ -1,141 +0,0 @@
using Azure;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq;
using System.Globalization;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class CopyProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
public CopyProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void process(Layer sourceImportWorker, Guid? parentId)
{
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
}
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
}
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
}
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
}
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
string? year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == parentId)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = parentId,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T1";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Records = new List<Record>();
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
Layer dataSource = db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.First();
/*
processedLayer.Sources.Add(new ProcessSource
{
LayerId = processedLayer.Id,
SourceId = dataSource.Id
});
*/
List<Record> newRecords = new List<Record>();
foreach (Record source in dataSource.Records!)
{
Record record = new Record
{
Id = Guid.NewGuid(),
Code = source.Code,
Value1 = source.Value1,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
if (isNew)
{
db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}
}

294
WebAPI/dataProcessors/deaggregation.processor.cs
View File

@@ -1,294 +0,0 @@
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class DeaggregationProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
public DeaggregationProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
}
public void process(Layer sourceImportWorker, Guid? parentId)
{
string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
}
string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
}
string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
}
string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
}
string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
}
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
}
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == parentId)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = parentId,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T2";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
List<Layer> dataSources = db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
&& x.CreatedAt.Month == int.Parse(month)
&& x.CreatedAt.Year == year
&& !x.IsDeleted)
.OrderBy(x => x.CreatedAt)
.ToList();
/*
foreach (Layer source in dataSources)
{
response.Sources.Add(new ProcessSource
{
LayerId = response.Id,
SourceId = source.Id
});
}
*/
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.First()?.Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = baseRecord.Code,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
float previousValue = 0;
//day 1
float firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
setValue(processedRecord, 1, firstVal);
previousValue = firstVal;
//days 2-29/30
for (int i=2; i<lastDayInMonth; i++)
{
float? dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (dayVal == null)
{
//TODO: missing day value? Should I log it?
setValue(processedRecord, i, 0);
} else
{
float processedVal = (dayVal ?? 0) - previousValue;
setValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0;
}
}
//last day
float? lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (lastVal == null)
{
setValue(processedRecord, lastDayInMonth, 0);
} else
{
setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
}
newRecords.Add(processedRecord);
}
if (isNew)
{
db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
private void setValue(Record record, int number, float? value)
{
switch (number)
{
case 1:
record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
}
}

6
WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
View File

@@ -8,9 +8,9 @@ namespace WebAPI.dataProcessors
{
public class T3MultiSourceSummaryProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3MultiSourceSummaryProcessor(
AppDbContext _db,

6
WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs
View File

@@ -8,9 +8,9 @@ namespace WebAPI.dataProcessors
{
public class T3MultiSourceYearSummaryProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3MultiSourceYearSummaryProcessor(
AppDbContext _db,

7
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -11,9 +11,9 @@ namespace WebAPI.dataProcessors
{
public class T3SingleSourceProcessor
{
private AppDbContext db;
private SpreadsheetsResource.ValuesResource googleSheetValues;
private LayersController controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3SingleSourceProcessor(
AppDbContext _db,
@@ -155,7 +155,6 @@ namespace WebAPI.dataProcessors
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}