diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index cf48ad3..668863d 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -1,5 +1,6 @@ using System.Globalization; using System.Xml.Serialization; +using DiunaBIWebAPI.dataImporters; using Google.Apis.Sheets.v4; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; @@ -66,9 +67,11 @@ namespace WebAPI.Controllers { try { - Request.Headers.TryGetValue("userId", out var value); - Guid currentUserId = new Guid(value!); - return Ok(AddLayer(input, currentUserId).Id); + //Request.Headers.TryGetValue("userId", out var value); + //Guid currentUserId = new Guid(value!); + + + return Ok(); } catch (Exception e) { @@ -92,31 +95,6 @@ namespace WebAPI.Controllers } } [HttpGet] - [Route("parseGoogleSheet/{sheetId}")] - public IActionResult ParseGoogleSheet(string sheetId) - { - - string sheetName = "KOSZTY"; - - Layer layer = new Layer(); - layer.Source = "GoogleSheet"; - layer.Number = db.Layers.Count() + 1; - var parser = new googleSheetParser(googleSheetValues); - dynamic parsedSheet = parser.parse(sheetId); - layer.Records = parsedSheet.records; - layer.Name = $"W{layer.Number}-I-{sheetName}-{parsedSheet.date}-{DateTime.Now.ToString("yyyyMMddHHmm")}"; - - return Ok(layer); - } - [HttpPost] - [DisableRequestSizeLimit] - [Route("parseFile")] - public IActionResult ParseFile() - { - var parser = new csvParser(); - return Ok(parser.parse(Request.Form.Files[0])); - } - [HttpGet] [Route("exportToGoogleSheet/{id}")] public IActionResult ExportToGoogleSheet(Guid id) { @@ -138,44 +116,97 @@ namespace WebAPI.Controllers { return Unauthorized(); } - - List importWorkerLayers; - List layersToImport; ; - try - { - importWorkerLayers = db.Layers - .Include(x => x.Records) - .Where(x => x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker")) - .ToList(); - - layersToImport = new List(); - foreach (Layer layer in importWorkerLayers) - { - string startDate = layer.Records!.Where(x => x.Code == "StartDate").First().Desc1!; - string endDate = layer.Records!.Where(x => x.Code == "EndDate").First().Desc1!; - var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); - var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) - { - processImportWorker(layer); - } - } - } - catch (Exception e) - { + try + { + List importWorkerLayers; + importWorkerLayers = db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker") && + x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") + ) + .OrderBy(x => x.CreatedAt) + .ToList(); + + if (importWorkerLayers.Count() == 0) + { + logsController.AddEntry(new LogEntry + { + Title = "No Layers to import.", + Type = LogEntryType.info, + LogType = LogType.import, + CreatedAt = DateTime.UtcNow, + }); + return Ok(); + } + + foreach (Layer importWorker in importWorkerLayers) + { + try + { + string? startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; + if (startDate == null) + { + throw new Exception("Year record nod found"); + } + string? endDate = importWorker.Records!.Where(x => x.Code == "EndDate").First().Desc1; + if (endDate == null) + { + throw new Exception("Year record nod found"); + } + var startDateParsed = DateTime.ParseExact(startDate!, "yyyy.MM.dd", null); + var endDateParsed = DateTime.ParseExact(endDate!, "yyyy.MM.dd", null); + if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) + { + MorskaImporter importer = new MorskaImporter(db, googleSheetValues, this); + importer.import(importWorker); + + logsController.AddEntry(new LogEntry + { + Title = $"{importWorker!.Name}, {importWorker.Id}", + Type = LogEntryType.info, + LogType = LogType.import, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + } + else + { + logsController.AddEntry(new LogEntry + { + Title = $"{importWorker!.Name}, {importWorker.Id}", + Type = LogEntryType.warning, + LogType = LogType.import, + Message = "importLayer is out of date. Should be disabled. Not processed.", + CreatedAt = DateTime.UtcNow + }); + } + } catch(Exception e) + { + logsController.AddEntry(new LogEntry + { + Title = $"{importWorker!.Name}, {importWorker.Id}", + Type = LogEntryType.error, + LogType = LogType.import, + Message = e.ToString(), + CreatedAt = DateTime.UtcNow + }); + } + } + return Ok(); + } catch(Exception e) + { logsController.AddEntry(new LogEntry { - Title = "Import error", + Title = "Process error", Type = LogEntryType.error, LogType = LogType.import, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); - return BadRequest(e.ToString()); + return BadRequest(e.ToString()); } - return Ok(); } - [HttpGet] [Route("AutoProcess/{apiKey}/{type}")] [AllowAnonymous] @@ -344,92 +375,6 @@ namespace WebAPI.Controllers }); } } - - return Ok(); - - - - - - foreach (Layer processWorker in processWorkerLayers) - { - string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1; - if (processType == null) - { - logsController.AddEntry(new LogEntry - { - Title = "Process error", - Type = LogEntryType.error, - LogType = LogType.import, - Message = processWorker?.Name + " ProcessType record not found", - CreatedAt = DateTime.UtcNow - }); - return BadRequest(); - } - List? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList(); - if (sources != null && sources.Count() == 0) - { - logsController.AddEntry(new LogEntry - { - Title = "Process error", - Type = LogEntryType.error, - LogType = LogType.import, - Message = processWorker?.Name + " Source record not found", - CreatedAt = DateTime.UtcNow - }); - return BadRequest(); - } - - if (processType == "T3-SourceSummary") - { - if (processWorker != null) - { - //T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this); - // processor.process(processWorker); - } - } else if (processType == "T3-MultiSourceSummary") - { - var tst = 5; - } else - { - - Layer sourceLayer = db.Layers.Include(x => x.Records) - .Single(x => x.Name == sources!.First().Desc1); - if (sourceLayer == null) - { - logsController.AddEntry(new LogEntry - { - Title = "Process error", - Type = LogEntryType.error, - LogType = LogType.import, - Message = processWorker?.Name + " Source layer not found " + sourceLayer, - CreatedAt = DateTime.UtcNow - }); - return BadRequest(); - } - - string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!; - string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!; - var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); - var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) - { - - logsController.AddEntry(new LogEntry - { - Title = $"Process Success, {sourceLayer.Name}", - Type = LogEntryType.info, - LogType = LogType.process, - CreatedAt = DateTime.UtcNow - }); - } else if (endDateParsed.Date >= DateTime.UtcNow.Date) - { - // TODO: make isEnabled = false - layer is out of date - var name = processWorker.Name; - } - } - } - return Ok(); } catch (Exception e) @@ -445,127 +390,6 @@ namespace WebAPI.Controllers return BadRequest(e.ToString()); } } - - - [HttpGet] - [Route("checkDates")] - public IActionResult checkDates() - { - var warsawTZ = TimeZoneInfo.FindSystemTimeZoneById("Singapore Standard Time"); - DateTime date = DateTime.UtcNow; - return Ok(date); - } - - // - private void processImportWorker(Layer importWorker) - { - // get and check settings - string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetId not found, {importWorker.Name}"); - } - string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; - if (sheetTabName == null) - { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); - } - string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; - if (importYearCell == null) - { - throw new Exception($"ImportYear not found, {importWorker.Name}"); - } - string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; - if (importMonthCell == null) - { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); - } - string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; - if (importNameCell == null) - { - throw new Exception($"ImportName not found, {importWorker.Name}"); - } - string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; - if (checkSumCell == null) - { - throw new Exception($"SheetId not found, {importWorker.Name}"); - } - string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1; - if (dataRange == null) - { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } - // open excel and read data - var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); - string? name = nameResponse.Values?[0][0].ToString(); - if (name == null) - { - throw new Exception($"ImportName cell is empty, {importWorker.Name}"); - } - var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); - string? year = yearResponse.Values[0][0].ToString(); - if (year == null) - { - throw new Exception($"ImportYear cell is empty, {importWorker.Name}"); - } - var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute(); - string? month = monthResponse.Values[0][0].ToString(); - if (month == null) - { - throw new Exception($"ImportMonth cell is empty, {importWorker.Name}"); - } - Layer layer = new Layer - { - Source = "GoogleSheet", - Number = db.Layers.Count() + 1, - ParentId = importWorker.Id - }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}"; - layer.Type = LayerType.import; - layer.Records = new List(); - var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - for (int i = 0; i < data[1].Count; i++) - { - float value; - if ( - data[0][i].ToString()?.Length > 0 && - float.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value)) - { - Record record = new Record - { - Id = Guid.NewGuid(), - Code = data[0][i].ToString(), - Value1 = value, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - layer.Records.Add(record); - }; - } - - AddLayer(layer, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - - logsController.AddEntry(new LogEntry - { - Title = $"Import Success, {importWorker.Name}", - Type = LogEntryType.info, - LogType = LogType.import, - CreatedAt = DateTime.UtcNow - }); - - } - internal Layer AddLayer(Layer input, Guid currentUserId) - { - input.CreatedById = currentUserId; - input.ModifiedById = currentUserId; - input.CreatedAt = DateTime.UtcNow; - input.ModifiedAt = DateTime.UtcNow; - db.Layers.Add(input); - SaveRecords(input.Id, input.Records!, currentUserId); - db.SaveChanges(); - return input; - } internal void SaveRecords(Guid id, ICollection records, Guid currentUserId) { try @@ -590,20 +414,5 @@ namespace WebAPI.Controllers throw; } } - private void SaveSources(ICollection sources) - { - try - { - foreach (ProcessSource processSource in sources) - { - Console.WriteLine($"{processSource.LayerId} {processSource.SourceId}"); - db.ProcessSources.Add(processSource); - } - } - catch (Exception) - { - throw; - } - } } } \ No newline at end of file diff --git a/WebAPI/DiunaBI-WebAPI.csproj b/WebAPI/DiunaBI-WebAPI.csproj index 4100186..d4b03f3 100644 --- a/WebAPI/DiunaBI-WebAPI.csproj +++ b/WebAPI/DiunaBI-WebAPI.csproj @@ -33,8 +33,10 @@ + + diff --git a/WebAPI/dataImporters/morska.importer.cs b/WebAPI/dataImporters/morska.importer.cs new file mode 100644 index 0000000..bd2c9ee --- /dev/null +++ b/WebAPI/dataImporters/morska.importer.cs @@ -0,0 +1,124 @@ +using System; +using System.Globalization; +using Google.Apis.Sheets.v4; +using WebAPI; +using WebAPI.Controllers; +using WebAPI.Models; + +namespace DiunaBIWebAPI.dataImporters +{ + public class MorskaImporter + { + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; + + public MorskaImporter( + AppDbContext _db, + SpreadsheetsResource.ValuesResource _googleSheetValues, + LayersController _controller) + { + db = _db; + googleSheetValues = _googleSheetValues; + controller = _controller; + } + + public void import(Layer importWorker) + { + string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; + if (sheetId == null) + { + throw new Exception($"SheetId not found, {importWorker.Name}"); + } + string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; + if (sheetTabName == null) + { + throw new Exception($"SheetTabName not found, {importWorker.Name}"); + } + string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; + if (importYearCell == null) + { + throw new Exception($"ImportYear not found, {importWorker.Name}"); + } + string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; + if (importMonthCell == null) + { + throw new Exception($"ImportMonth not found, {importWorker.Name}"); + } + string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; + if (importNameCell == null) + { + throw new Exception($"ImportName not found, {importWorker.Name}"); + } + string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; + if (checkSumCell == null) + { + throw new Exception($"SheetId not found, {importWorker.Name}"); + } + string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1; + if (dataRange == null) + { + throw new Exception($"DataRange not found, {importWorker.Name}"); + } + + var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); + string? name = nameResponse.Values?[0][0].ToString(); + if (name == null) + { + throw new Exception($"ImportName cell is empty, {importWorker.Name}"); + } + var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); + string? year = yearResponse.Values[0][0].ToString(); + if (year == null) + { + throw new Exception($"ImportYear cell is empty, {importWorker.Name}"); + } + var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute(); + string? month = monthResponse.Values[0][0].ToString(); + if (month == null) + { + throw new Exception($"ImportMonth cell is empty, {importWorker.Name}"); + } + Layer layer = new Layer + { + Source = "GoogleSheet", + Number = db.Layers.Count() + 1, + ParentId = importWorker.Id, + Type = LayerType.import, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}"; + + List newRecords = new List(); + + var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var data = dataRangeResponse.Values; + for (int i = 0; i < data[1].Count; i++) + { + float value; + if ( + data[0][i].ToString()?.Length > 0 && + float.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value)) + { + Record record = new Record + { + Id = Guid.NewGuid(), + Code = data[0][i].ToString(), + Value1 = value, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + newRecords.Add(record); + }; + } + + db.Layers.Add(layer); + controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + db.SaveChanges(); + } + } +} + diff --git a/WebAPI/dataProcessors/copy.processor.cs b/WebAPI/dataProcessors/copy.processor.cs deleted file mode 100644 index dfcfd45..0000000 --- a/WebAPI/dataProcessors/copy.processor.cs +++ /dev/null @@ -1,141 +0,0 @@ -using Azure; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; -using Microsoft.EntityFrameworkCore; -using Newtonsoft.Json.Linq; -using System.Globalization; -using WebAPI.Controllers; -using WebAPI.Models; - -namespace WebAPI.dataProcessors -{ - public class CopyProcessor - { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; - - public CopyProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) - { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - } - - public void process(Layer sourceImportWorker, Guid? parentId) - { - string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetId not found, {sourceImportWorker.Name}"); - } - string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}"); - } - string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; - if (importYearCell == null) - { - throw new Exception($"ImportYear not found, {sourceImportWorker.Name}"); - } - string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; - if (importMonthCell == null) - { - throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}"); - } - string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; - if (importNameCell == null) - { - throw new Exception($"ImportName not found, {sourceImportWorker.Name}"); - } - var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); - string? name = nameResponse.Values[0][0].ToString(); - if (name == null) - { - throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}"); - } - var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); - string? year = yearResponse.Values[0][0].ToString(); - if (year == null) - { - throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}"); - } - var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute(); - string? month = monthResponse.Values[0][0].ToString(); - if (month == null) - { - throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}"); - } - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == parentId) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = parentId, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.Records = new List(); - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - Layer dataSource = db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .First(); - /* - processedLayer.Sources.Add(new ProcessSource - { - LayerId = processedLayer.Id, - SourceId = dataSource.Id - }); - */ - - List newRecords = new List(); - foreach (Record source in dataSource.Records!) - { - Record record = new Record - { - Id = Guid.NewGuid(), - Code = source.Code, - Value1 = source.Value1, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - newRecords.Add(record); - } - if (isNew) - { - db.Layers.Add(processedLayer); - } else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); - } - } -} diff --git a/WebAPI/dataProcessors/deaggregation.processor.cs b/WebAPI/dataProcessors/deaggregation.processor.cs deleted file mode 100644 index e792b06..0000000 --- a/WebAPI/dataProcessors/deaggregation.processor.cs +++ /dev/null @@ -1,294 +0,0 @@ -using Google.Apis.Sheets.v4; -using Microsoft.AspNetCore.Mvc; -using Microsoft.EntityFrameworkCore; -using System; -using System.Globalization; -using WebAPI.Controllers; -using WebAPI.Models; - -namespace WebAPI.dataProcessors -{ - public class DeaggregationProcessor - { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; - - public DeaggregationProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) - { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - } - - public void process(Layer sourceImportWorker, Guid? parentId) - { - string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetId not found, {sourceImportWorker.Name}"); - } - string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}"); - } - string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; - if (importYearCell == null) - { - throw new Exception($"ImportYear not found, {sourceImportWorker.Name}"); - } - string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; - if (importMonthCell == null) - { - throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}"); - } - string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; - if (importNameCell == null) - { - throw new Exception($"ImportName not found, {sourceImportWorker.Name}"); - } - var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); - string? name = nameResponse.Values[0][0].ToString(); - if (name == null) - { - throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}"); - } - var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); - int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? ""); - if (year == null) - { - throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}"); - } - var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute(); - string? month = monthResponse.Values[0][0].ToString(); - if (month == null) - { - throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}"); - } - - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == parentId) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = parentId, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T2"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - List newRecords = new List(); - - List dataSources = db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && x.CreatedAt.Month == int.Parse(month) - && x.CreatedAt.Year == year - && !x.IsDeleted) - .OrderBy(x => x.CreatedAt) - .ToList(); - - /* - foreach (Layer source in dataSources) - { - response.Sources.Add(new ProcessSource - { - LayerId = response.Id, - SourceId = source.Id - }); - } - */ - - List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (Record baseRecord in dataSources.First()?.Records!) - { - List codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - - Record processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? "")); - float previousValue = 0; - //day 1 - float firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1)) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1 ?? 0; - setValue(processedRecord, 1, firstVal); - previousValue = firstVal; - //days 2-29/30 - for (int i=2; i x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? "")) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1; - if (dayVal == null) - { - //TODO: missing day value? Should I log it? - setValue(processedRecord, i, 0); - } else - { - float processedVal = (dayVal ?? 0) - previousValue; - setValue(processedRecord, i, processedVal); - previousValue = dayVal ?? 0; - } - } - //last day - float? lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth)) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1; - - if (lastVal == null) - { - setValue(processedRecord, lastDayInMonth, 0); - } else - { - setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue); - } - - newRecords.Add(processedRecord); - } - - if (isNew) - { - db.Layers.Add(processedLayer); - } else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); - } - private void setValue(Record record, int number, float? value) - { - switch (number) - { - case 1: - record.Value1 = value; - break; - case 2: - record.Value2 = value; - break; - case 3: - record.Value3 = value; - break; - case 4: - record.Value4 = value; - break; - case 5: - record.Value5 = value; - break; - case 6: - record.Value6 = value; - break; - case 7: - record.Value7 = value; - break; - case 8: - record.Value8 = value; - break; - case 9: - record.Value9 = value; - break; - case 10: - record.Value10 = value; - break; - case 11: - record.Value11 = value; - break; - case 12: - record.Value12 = value; - break; - case 13: - record.Value13 = value; - break; - case 14: - record.Value14 = value; - break; - case 15: - record.Value15 = value; - break; - case 16: - record.Value16 = value; - break; - case 17: - record.Value17 = value; - break; - case 18: - record.Value18 = value; - break; - case 19: - record.Value19 = value; - break; - case 20: - record.Value20 = value; - break; - case 21: - record.Value21 = value; - break; - case 22: - record.Value22 = value; - break; - case 23: - record.Value23 = value; - break; - case 24: - record.Value24 = value; - break; - case 25: - record.Value25 = value; - break; - case 26: - record.Value26 = value; - break; - case 27: - record.Value27 = value; - break; - case 28: - record.Value28 = value; - break; - case 29: - record.Value29 = value; - break; - case 30: - record.Value30 = value; - break; - case 31: - record.Value31 = value; - break; - case 32: - record.Value32 = value; - break; - } - } - } -} diff --git a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs index edfa828..d94c299 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs @@ -8,9 +8,9 @@ namespace WebAPI.dataProcessors { public class T3MultiSourceSummaryProcessor { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; public T3MultiSourceSummaryProcessor( AppDbContext _db, diff --git a/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs index cb59b4a..df922e8 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs @@ -8,9 +8,9 @@ namespace WebAPI.dataProcessors { public class T3MultiSourceYearSummaryProcessor { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; public T3MultiSourceYearSummaryProcessor( AppDbContext _db, diff --git a/WebAPI/dataProcessors/t3.SingleSource.processor.cs b/WebAPI/dataProcessors/t3.SingleSource.processor.cs index 0e1cf37..470cb38 100644 --- a/WebAPI/dataProcessors/t3.SingleSource.processor.cs +++ b/WebAPI/dataProcessors/t3.SingleSource.processor.cs @@ -11,9 +11,9 @@ namespace WebAPI.dataProcessors { public class T3SingleSourceProcessor { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; public T3SingleSourceProcessor( AppDbContext _db, @@ -155,7 +155,6 @@ namespace WebAPI.dataProcessors } controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); db.SaveChanges(); - } }