AutoImport refactor
This commit is contained in:
Michał Zieliński
@@ -1,5 +1,6 @@
|
||||
using System.Globalization;
|
||||
using System.Xml.Serialization;
|
||||
using DiunaBIWebAPI.dataImporters;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
@@ -66,9 +67,11 @@ namespace WebAPI.Controllers
|
||||
{
|
||||
try
|
||||
{
|
||||
Request.Headers.TryGetValue("userId", out var value);
|
||||
Guid currentUserId = new Guid(value!);
|
||||
return Ok(AddLayer(input, currentUserId).Id);
|
||||
//Request.Headers.TryGetValue("userId", out var value);
|
||||
//Guid currentUserId = new Guid(value!);
|
||||
|
||||
|
||||
return Ok();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
@@ -92,31 +95,6 @@ namespace WebAPI.Controllers
|
||||
}
|
||||
}
|
||||
[HttpGet]
|
||||
[Route("parseGoogleSheet/{sheetId}")]
|
||||
public IActionResult ParseGoogleSheet(string sheetId)
|
||||
{
|
||||
|
||||
string sheetName = "KOSZTY";
|
||||
|
||||
Layer layer = new Layer();
|
||||
layer.Source = "GoogleSheet";
|
||||
layer.Number = db.Layers.Count() + 1;
|
||||
var parser = new googleSheetParser(googleSheetValues);
|
||||
dynamic parsedSheet = parser.parse(sheetId);
|
||||
layer.Records = parsedSheet.records;
|
||||
layer.Name = $"W{layer.Number}-I-{sheetName}-{parsedSheet.date}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
|
||||
|
||||
return Ok(layer);
|
||||
}
|
||||
[HttpPost]
|
||||
[DisableRequestSizeLimit]
|
||||
[Route("parseFile")]
|
||||
public IActionResult ParseFile()
|
||||
{
|
||||
var parser = new csvParser();
|
||||
return Ok(parser.parse(Request.Form.Files[0]));
|
||||
}
|
||||
[HttpGet]
|
||||
[Route("exportToGoogleSheet/{id}")]
|
||||
public IActionResult ExportToGoogleSheet(Guid id)
|
||||
{
|
||||
@@ -138,44 +116,97 @@ namespace WebAPI.Controllers
|
||||
{
|
||||
return Unauthorized();
|
||||
}
|
||||
|
||||
List<Layer> importWorkerLayers;
|
||||
List<Layer> layersToImport; ;
|
||||
try
|
||||
{
|
||||
importWorkerLayers = db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker"))
|
||||
.ToList();
|
||||
|
||||
layersToImport = new List<Layer>();
|
||||
foreach (Layer layer in importWorkerLayers)
|
||||
{
|
||||
string startDate = layer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
|
||||
string endDate = layer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
processImportWorker(layer);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
try
|
||||
{
|
||||
List<Layer> importWorkerLayers;
|
||||
importWorkerLayers = db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x =>
|
||||
x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ImportWorker") &&
|
||||
x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
|
||||
)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToList();
|
||||
|
||||
if (importWorkerLayers.Count() == 0)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "No Layers to import.",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.import,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
});
|
||||
return Ok();
|
||||
}
|
||||
|
||||
foreach (Layer importWorker in importWorkerLayers)
|
||||
{
|
||||
try
|
||||
{
|
||||
string? startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
|
||||
if (startDate == null)
|
||||
{
|
||||
throw new Exception("Year record nod found");
|
||||
}
|
||||
string? endDate = importWorker.Records!.Where(x => x.Code == "EndDate").First().Desc1;
|
||||
if (endDate == null)
|
||||
{
|
||||
throw new Exception("Year record nod found");
|
||||
}
|
||||
var startDateParsed = DateTime.ParseExact(startDate!, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate!, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
MorskaImporter importer = new MorskaImporter(db, googleSheetValues, this);
|
||||
importer.import(importWorker);
|
||||
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker!.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.import,
|
||||
Message = "Success",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker!.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.warning,
|
||||
LogType = LogType.import,
|
||||
Message = "importLayer is out of date. Should be disabled. Not processed.",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
} catch(Exception e)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker!.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = e.ToString(),
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
}
|
||||
return Ok();
|
||||
} catch(Exception e)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Import error",
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = e.ToString(),
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest(e.ToString());
|
||||
return BadRequest(e.ToString());
|
||||
}
|
||||
return Ok();
|
||||
}
|
||||
|
||||
[HttpGet]
|
||||
[Route("AutoProcess/{apiKey}/{type}")]
|
||||
[AllowAnonymous]
|
||||
@@ -344,92 +375,6 @@ namespace WebAPI.Controllers
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return Ok();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
foreach (Layer processWorker in processWorkerLayers)
|
||||
{
|
||||
string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
|
||||
if (processType == null)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = processWorker?.Name + " ProcessType record not found",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
}
|
||||
List<Record>? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList();
|
||||
if (sources != null && sources.Count() == 0)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = processWorker?.Name + " Source record not found",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
}
|
||||
|
||||
if (processType == "T3-SourceSummary")
|
||||
{
|
||||
if (processWorker != null)
|
||||
{
|
||||
//T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
|
||||
// processor.process(processWorker);
|
||||
}
|
||||
} else if (processType == "T3-MultiSourceSummary")
|
||||
{
|
||||
var tst = 5;
|
||||
} else
|
||||
{
|
||||
|
||||
Layer sourceLayer = db.Layers.Include(x => x.Records)
|
||||
.Single(x => x.Name == sources!.First().Desc1);
|
||||
if (sourceLayer == null)
|
||||
{
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = "Process error",
|
||||
Type = LogEntryType.error,
|
||||
LogType = LogType.import,
|
||||
Message = processWorker?.Name + " Source layer not found " + sourceLayer,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
return BadRequest();
|
||||
}
|
||||
|
||||
string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
|
||||
string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"Process Success, {sourceLayer.Name}",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.process,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
} else if (endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
// TODO: make isEnabled = false - layer is out of date
|
||||
var name = processWorker.Name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Ok();
|
||||
}
|
||||
catch (Exception e)
|
||||
@@ -445,127 +390,6 @@ namespace WebAPI.Controllers
|
||||
return BadRequest(e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
[HttpGet]
|
||||
[Route("checkDates")]
|
||||
public IActionResult checkDates()
|
||||
{
|
||||
var warsawTZ = TimeZoneInfo.FindSystemTimeZoneById("Singapore Standard Time");
|
||||
DateTime date = DateTime.UtcNow;
|
||||
return Ok(date);
|
||||
}
|
||||
|
||||
//
|
||||
private void processImportWorker(Layer importWorker)
|
||||
{
|
||||
// get and check settings
|
||||
string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||
}
|
||||
string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
|
||||
if (sheetTabName == null)
|
||||
{
|
||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||
}
|
||||
string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
|
||||
if (importYearCell == null)
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||
}
|
||||
string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
|
||||
if (importMonthCell == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||
}
|
||||
string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
|
||||
if (importNameCell == null)
|
||||
{
|
||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
||||
}
|
||||
string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
|
||||
if (checkSumCell == null)
|
||||
{
|
||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||
}
|
||||
string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1;
|
||||
if (dataRange == null)
|
||||
{
|
||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||
}
|
||||
// open excel and read data
|
||||
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
|
||||
string? name = nameResponse.Values?[0][0].ToString();
|
||||
if (name == null)
|
||||
{
|
||||
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
|
||||
}
|
||||
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
|
||||
string? year = yearResponse.Values[0][0].ToString();
|
||||
if (year == null)
|
||||
{
|
||||
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
|
||||
}
|
||||
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
|
||||
string? month = monthResponse.Values[0][0].ToString();
|
||||
if (month == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
|
||||
}
|
||||
Layer layer = new Layer
|
||||
{
|
||||
Source = "GoogleSheet",
|
||||
Number = db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
|
||||
layer.Type = LayerType.import;
|
||||
layer.Records = new List<Record>();
|
||||
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||
var data = dataRangeResponse.Values;
|
||||
for (int i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
float value;
|
||||
if (
|
||||
data[0][i].ToString()?.Length > 0 &&
|
||||
float.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
|
||||
{
|
||||
Record record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = data[0][i].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Records.Add(record);
|
||||
};
|
||||
}
|
||||
|
||||
AddLayer(layer, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
||||
|
||||
logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"Import Success, {importWorker.Name}",
|
||||
Type = LogEntryType.info,
|
||||
LogType = LogType.import,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
|
||||
}
|
||||
internal Layer AddLayer(Layer input, Guid currentUserId)
|
||||
{
|
||||
input.CreatedById = currentUserId;
|
||||
input.ModifiedById = currentUserId;
|
||||
input.CreatedAt = DateTime.UtcNow;
|
||||
input.ModifiedAt = DateTime.UtcNow;
|
||||
db.Layers.Add(input);
|
||||
SaveRecords(input.Id, input.Records!, currentUserId);
|
||||
db.SaveChanges();
|
||||
return input;
|
||||
}
|
||||
internal void SaveRecords(Guid id, ICollection<Record> records, Guid currentUserId)
|
||||
{
|
||||
try
|
||||
@@ -590,20 +414,5 @@ namespace WebAPI.Controllers
|
||||
throw;
|
||||
}
|
||||
}
|
||||
private void SaveSources(ICollection<ProcessSource> sources)
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (ProcessSource processSource in sources)
|
||||
{
|
||||
Console.WriteLine($"{processSource.LayerId} {processSource.SourceId}");
|
||||
db.ProcessSources.Add(processSource);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user