WIP: D1 importeer
This commit is contained in:
Michał Zieliński
BIN
WebAPI/.DS_Store
vendored
BIN
WebAPI/.DS_Store
vendored
Binary file not shown.
@@ -279,6 +279,7 @@ public class LayersController : Controller
|
||||
.Where(x =>
|
||||
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
||||
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
|
||||
&& x.Number == 2989
|
||||
)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.ToList();
|
||||
@@ -302,43 +303,11 @@ public class LayersController : Controller
|
||||
{
|
||||
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ??
|
||||
"Standard";
|
||||
|
||||
if (type == "FK2")
|
||||
switch (type)
|
||||
{
|
||||
var importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
|
||||
importer.Import(importWorker);
|
||||
Thread.Sleep(5000); // be aware of GSheet API quota
|
||||
|
||||
_logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Info,
|
||||
LogType = LogType.Import,
|
||||
Message = "Success",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
|
||||
if (startDate == null)
|
||||
{
|
||||
throw new Exception("StartDate record nod found");
|
||||
}
|
||||
|
||||
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
|
||||
if (endDate == null)
|
||||
{
|
||||
throw new Exception("EndDate record nod found");
|
||||
}
|
||||
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date &&
|
||||
endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
var importer = new MorskaImporter(_db, _googleSheetValues, this);
|
||||
importer.Import(importWorker);
|
||||
case "D1":
|
||||
var d1Importer = new MorskaD1Importer(_db, _googleSheetValues, this);
|
||||
d1Importer.Import(importWorker);
|
||||
Thread.Sleep(5000); // be aware of GSheet API quota
|
||||
|
||||
_logsController.AddEntry(new LogEntry
|
||||
@@ -349,32 +318,83 @@ public class LayersController : Controller
|
||||
Message = "Success",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else if (IsImportedLayerUpToDate(importWorker) == false)
|
||||
break;
|
||||
case "FK2":
|
||||
{
|
||||
var importer = new MorskaImporter(_db, _googleSheetValues, this);
|
||||
importer.Import(importWorker);
|
||||
var fk2Importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
|
||||
fk2Importer.Import(importWorker);
|
||||
Thread.Sleep(5000); // be aware of GSheet API quota
|
||||
|
||||
_logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Warning,
|
||||
Type = LogEntryType.Info,
|
||||
LogType = LogType.Import,
|
||||
Message = "Success (reimported)",
|
||||
Message = "Success",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
break;
|
||||
}
|
||||
else
|
||||
default:
|
||||
{
|
||||
_logsController.AddEntry(new LogEntry
|
||||
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
|
||||
if (startDate == null)
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Warning,
|
||||
LogType = LogType.Import,
|
||||
Message = "importLayer records are up of date. Not processed.",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
throw new Exception("StartDate record nod found");
|
||||
}
|
||||
|
||||
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
|
||||
if (endDate == null)
|
||||
{
|
||||
throw new Exception("EndDate record nod found");
|
||||
}
|
||||
|
||||
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
|
||||
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
|
||||
if (startDateParsed.Date <= DateTime.UtcNow.Date &&
|
||||
endDateParsed.Date >= DateTime.UtcNow.Date)
|
||||
{
|
||||
var importer = new MorskaImporter(_db, _googleSheetValues, this);
|
||||
importer.Import(importWorker);
|
||||
Thread.Sleep(5000); // be aware of GSheet API quota
|
||||
|
||||
_logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Info,
|
||||
LogType = LogType.Import,
|
||||
Message = "Success",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else if (IsImportedLayerUpToDate(importWorker) == false)
|
||||
{
|
||||
var importer = new MorskaImporter(_db, _googleSheetValues, this);
|
||||
importer.Import(importWorker);
|
||||
Thread.Sleep(5000); // be aware of GSheet API quota
|
||||
|
||||
_logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Warning,
|
||||
LogType = LogType.Import,
|
||||
Message = "Success (reimported)",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
_logsController.AddEntry(new LogEntry
|
||||
{
|
||||
Title = $"{importWorker.Name}, {importWorker.Id}",
|
||||
Type = LogEntryType.Warning,
|
||||
LogType = LogType.Import,
|
||||
Message = "importLayer records are up of date. Not processed.",
|
||||
CreatedAt = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
120
WebAPI/dataImporters/morska.d1.importer.cs
Normal file
120
WebAPI/dataImporters/morska.d1.importer.cs
Normal file
@@ -0,0 +1,120 @@
|
||||
using System.Globalization;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using WebAPI;
|
||||
using WebAPI.Controllers;
|
||||
using WebAPI.Models;
|
||||
|
||||
namespace DiunaBIWebAPI.dataImporters;
|
||||
|
||||
public class MorskaD1Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
LayersController controller)
|
||||
{
|
||||
public void Import(Layer importWorker)
|
||||
{
|
||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
||||
if (sheetId == null)
|
||||
{
|
||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||
}
|
||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
||||
if (sheetTabName == null)
|
||||
{
|
||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||
}
|
||||
var importYearCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||
if (importYearCell == null)
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||
}
|
||||
var importMonthCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||
if (importMonthCell == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||
}
|
||||
var importNameCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||
if (importNameCell == null)
|
||||
{
|
||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
||||
}
|
||||
|
||||
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
|
||||
if (dataRange == null)
|
||||
{
|
||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||
}
|
||||
|
||||
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
|
||||
var name = nameResponse.Values?[0][0].ToString();
|
||||
if (name == null)
|
||||
{
|
||||
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
|
||||
}
|
||||
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
|
||||
var year = yearResponse.Values[0][0].ToString();
|
||||
if (year == null)
|
||||
{
|
||||
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
|
||||
}
|
||||
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
|
||||
var month = monthResponse.Values[0][0].ToString();
|
||||
if (month == null)
|
||||
{
|
||||
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
|
||||
}
|
||||
var layer = new Layer
|
||||
{
|
||||
Number = db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
|
||||
|
||||
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||
var data = dataRangeResponse.Values;
|
||||
var newRecords = (from t in data
|
||||
where t.Count > 13 && (string)t[0] != string.Empty
|
||||
select new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = t[0].ToString(),
|
||||
Value1 = ParseValue(t[3].ToString()),
|
||||
Value2 = ParseValue(t[4].ToString()),
|
||||
Value3 = ParseValue(t[5].ToString()),
|
||||
Value4 = ParseValue(t[6].ToString()),
|
||||
Value5 = ParseValue(t[7].ToString()),
|
||||
Value6 = ParseValue(t[8].ToString()),
|
||||
Value7 = ParseValue(t[9].ToString()),
|
||||
Value8 = ParseValue(t[10].ToString()),
|
||||
Value9 = ParseValue(t[11].ToString()),
|
||||
Value10 = ParseValue(t[12].ToString()),
|
||||
Value11 = ParseValue(t[13].ToString()),
|
||||
Value12 = ParseValue(t[14].ToString()),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
}).ToList();
|
||||
db.Layers.Add(layer);
|
||||
controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
private double? ParseValue(string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
|
||||
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
|
||||
try
|
||||
{
|
||||
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
||||
return result;
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user