imports: save records
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Database.Context;
|
||||||
|
|
||||||
@@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaD1Importer> _logger;
|
||||||
|
|
||||||
public MorskaD1Importer(
|
public MorskaD1Importer(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues)
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaD1Importer> logger)
|
||||||
{
|
{
|
||||||
_db = db;
|
_db = db;
|
||||||
_googleSheetValues = googleSheetValues;
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
|
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
||||||
if (sheetId == null)
|
if (sheetId == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
||||||
if (sheetTabName == null)
|
if (sheetTabName == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||||
if (year == null)
|
if (year == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||||
if (month == null)
|
if (month == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||||
if (name == null)
|
if (name == null)
|
||||||
{
|
{
|
||||||
@@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
sheetId, sheetTabName, dataRange);
|
||||||
|
|
||||||
var layer = new Layer
|
var layer = new Layer
|
||||||
{
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
Number = _db.Layers.Count() + 1,
|
Number = _db.Layers.Count() + 1,
|
||||||
ParentId = importWorker.Id,
|
ParentId = importWorker.Id,
|
||||||
Type = LayerType.Import,
|
Type = LayerType.Import,
|
||||||
@@ -64,38 +79,74 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
CreatedAt = DateTime.UtcNow,
|
CreatedAt = DateTime.UtcNow,
|
||||||
ModifiedAt = DateTime.UtcNow
|
ModifiedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
|
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
try
|
||||||
var data = dataRangeResponse.Values;
|
{
|
||||||
var newRecords = (from t in data
|
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||||
where t.Count > 1 && (string)t[0] != string.Empty
|
var data = dataRangeResponse.Values;
|
||||||
select new Record
|
|
||||||
{
|
_logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = t[0].ToString(),
|
var newRecords = (from t in data
|
||||||
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
|
where t.Count > 1 && (string)t[0] != string.Empty
|
||||||
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
|
select new Record
|
||||||
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
|
{
|
||||||
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
|
Id = Guid.NewGuid(),
|
||||||
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
|
Code = t[0].ToString(),
|
||||||
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
|
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
|
||||||
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
|
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
|
||||||
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
|
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
|
||||||
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
|
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
|
||||||
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
|
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
|
||||||
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
|
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
|
||||||
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
|
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
|
||||||
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
|
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
|
||||||
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
|
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
|
||||||
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
|
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
|
||||||
CreatedAt = DateTime.UtcNow,
|
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
|
||||||
ModifiedAt = DateTime.UtcNow
|
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
|
||||||
}).ToList();
|
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
|
||||||
_db.Layers.Add(layer);
|
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
|
||||||
// TODO: Save records to the layer
|
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
|
||||||
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
CreatedAt = DateTime.UtcNow,
|
||||||
_db.SaveChanges();
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
|
SaveRecords(layer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
newRecords.Count, layer.Name, layer.Id);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private double? ParseValue(string? value)
|
private double? ParseValue(string? value)
|
||||||
@@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter
|
|||||||
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
catch (FormatException)
|
catch (FormatException e)
|
||||||
{
|
{
|
||||||
|
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IndexExists(IList<object> array, int index)
|
private bool IndexExists(IList<object> array, int index)
|
||||||
{
|
{
|
||||||
return array != null && index >= 0 && index < array.Count;
|
return array != null && index >= 0 && index < array.Count;
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Database.Context;
|
||||||
|
|
||||||
@@ -9,74 +10,136 @@ namespace DiunaBI.Plugins.Morska.Importers;
|
|||||||
public class MorskaD3Importer : MorskaBaseImporter
|
public class MorskaD3Importer : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaD3";
|
public override string ImporterType => "MorskaD3";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<MorskaD3Importer> _logger;
|
||||||
|
|
||||||
public MorskaD3Importer(
|
public MorskaD3Importer(
|
||||||
AppDbContext db)
|
AppDbContext db,
|
||||||
|
ILogger<MorskaD3Importer> logger)
|
||||||
{
|
{
|
||||||
_db = db;
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
|
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||||
if (year == null)
|
if (year == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||||
if (month == null)
|
if (month == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||||
if (name == null)
|
if (name == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
throw new Exception($"ImportName not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
|
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
|
||||||
if (name == null)
|
if (type == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportType not found, {importWorker.Name}");
|
throw new Exception($"ImportType not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
|
||||||
|
|
||||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
|
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
|
||||||
if (dataInbox == null)
|
if (dataInbox == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"DataInbox not found, {type}");
|
throw new Exception($"DataInbox not found, {type}");
|
||||||
}
|
}
|
||||||
var data = Convert.FromBase64String(dataInbox.Data);
|
|
||||||
var tst = Encoding.UTF8.GetString(data);
|
|
||||||
var records = JsonSerializer.Deserialize<List<Record>>(Encoding.UTF8.GetString(data));
|
|
||||||
if (records == null)
|
|
||||||
{
|
|
||||||
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
|
|
||||||
}
|
|
||||||
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
|
|
||||||
if (records.Count == 0)
|
|
||||||
{
|
|
||||||
throw new Exception($"No records found for {year}{month}");
|
|
||||||
}
|
|
||||||
records = records.Select(x =>
|
|
||||||
{
|
|
||||||
x.Id = Guid.NewGuid();
|
|
||||||
x.CreatedAt = DateTime.UtcNow;
|
|
||||||
x.ModifiedAt = DateTime.UtcNow;
|
|
||||||
return x;
|
|
||||||
}).ToList();
|
|
||||||
var layer = new Layer
|
|
||||||
{
|
|
||||||
Number = _db.Layers.Count() + 1,
|
|
||||||
ParentId = importWorker.Id,
|
|
||||||
Type = LayerType.Import,
|
|
||||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
||||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
|
|
||||||
|
|
||||||
_db.Layers.Add(layer);
|
_logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||||
// TODO: Save records to the layer
|
dataInbox.Id, dataInbox.CreatedAt);
|
||||||
//controller.SaveRecords(layer.Id, records, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
|
||||||
_db.SaveChanges();
|
try
|
||||||
|
{
|
||||||
|
var data = Convert.FromBase64String(dataInbox.Data);
|
||||||
|
var jsonString = Encoding.UTF8.GetString(data);
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
|
||||||
|
|
||||||
|
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
||||||
|
if (records == null)
|
||||||
|
{
|
||||||
|
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
|
||||||
|
|
||||||
|
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
|
||||||
|
if (records.Count == 0)
|
||||||
|
{
|
||||||
|
throw new Exception($"No records found for {year}{month}");
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||||
|
records.Count, year, month);
|
||||||
|
|
||||||
|
records = records.Select(x =>
|
||||||
|
{
|
||||||
|
x.Id = Guid.NewGuid();
|
||||||
|
x.CreatedAt = DateTime.UtcNow;
|
||||||
|
x.ModifiedAt = DateTime.UtcNow;
|
||||||
|
return x;
|
||||||
|
}).ToList();
|
||||||
|
|
||||||
|
var layer = new Layer
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Number = _db.Layers.Count() + 1,
|
||||||
|
ParentId = importWorker.Id,
|
||||||
|
Type = LayerType.Import,
|
||||||
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
|
SaveRecords(layer.Id, records);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
records.Count, layer.Name, layer.Id);
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Database.Context;
|
||||||
|
|
||||||
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
|
|||||||
public class MorskaFk2Importer : MorskaBaseImporter
|
public class MorskaFk2Importer : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaFK2";
|
public override string ImporterType => "MorskaFK2";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaFk2Importer> _logger;
|
||||||
|
|
||||||
public MorskaFk2Importer(
|
public MorskaFk2Importer(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues)
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaFk2Importer> logger)
|
||||||
{
|
{
|
||||||
_db = db;
|
_db = db;
|
||||||
_googleSheetValues = googleSheetValues;
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
|
_logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
||||||
if (sheetId == null)
|
if (sheetId == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
||||||
if (sheetTabName == null)
|
if (sheetTabName == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||||
if (year == null)
|
if (year == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||||
if (month == null)
|
if (month == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||||
if (name == null)
|
if (name == null)
|
||||||
{
|
{
|
||||||
@@ -52,8 +65,12 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
sheetId, sheetTabName, dataRange);
|
||||||
|
|
||||||
var layer = new Layer
|
var layer = new Layer
|
||||||
{
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
Number = _db.Layers.Count() + 1,
|
Number = _db.Layers.Count() + 1,
|
||||||
ParentId = importWorker.Id,
|
ParentId = importWorker.Id,
|
||||||
Type = LayerType.Import,
|
Type = LayerType.Import,
|
||||||
@@ -62,40 +79,91 @@ public class MorskaFk2Importer : MorskaBaseImporter
|
|||||||
CreatedAt = DateTime.UtcNow,
|
CreatedAt = DateTime.UtcNow,
|
||||||
ModifiedAt = DateTime.UtcNow
|
ModifiedAt = DateTime.UtcNow
|
||||||
};
|
};
|
||||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
|
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
||||||
|
|
||||||
var newRecords = new List<Record>();
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
try
|
||||||
var data = dataRangeResponse.Values;
|
|
||||||
for (var i = 0; i < data.Count; i++)
|
|
||||||
{
|
{
|
||||||
if (data[i].Count <= 9 || (string)data[i][3] == string.Empty) continue;
|
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||||
var dateArr = data[i][1].ToString()!.Split(".");
|
var data = dataRangeResponse.Values;
|
||||||
if (dateArr.Length != 3)
|
|
||||||
|
_logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
||||||
|
|
||||||
|
if (data != null)
|
||||||
{
|
{
|
||||||
throw new Exception($"Invalid date in row {i}");
|
for (var i = 0; i < data.Count; i++)
|
||||||
|
{
|
||||||
|
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var dateArr = data[i][1].ToString()!.Split(".");
|
||||||
|
if (dateArr.Length != 3)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]);
|
||||||
|
throw new Exception($"Invalid date in row {i}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var number = data[i][2].ToString()!;
|
||||||
|
if (number.Length == 1) number = $"0{number}";
|
||||||
|
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
||||||
|
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||||
|
{
|
||||||
|
_logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var record = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = code,
|
||||||
|
Desc1 = data[i][3].ToString(),
|
||||||
|
Value1 = value,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
newRecords.Add(record);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var number = data[i][2].ToString()!;
|
_db.Layers.Add(layer);
|
||||||
if (number.Length == 1) number = $"0{number}";
|
|
||||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
SaveRecords(layer.Id, newRecords);
|
||||||
if (!(data[i][9].ToString()?.Length > 0) ||
|
_db.SaveChanges();
|
||||||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
|
|
||||||
var record = new Record
|
_logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
{
|
newRecords.Count, layer.Name, layer.Id);
|
||||||
Id = Guid.NewGuid(),
|
|
||||||
Code = code,
|
|
||||||
Desc1 = data[i][3].ToString(),
|
|
||||||
Value1 = value,
|
|
||||||
CreatedAt = DateTime.UtcNow,
|
|
||||||
ModifiedAt = DateTime.UtcNow
|
|
||||||
};
|
|
||||||
newRecords.Add(record);
|
|
||||||
}
|
}
|
||||||
_db.Layers.Add(layer);
|
catch (Exception e)
|
||||||
// TODO: Save records to the layer
|
{
|
||||||
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
_logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId);
|
||||||
_db.SaveChanges();
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Database.Context;
|
||||||
|
|
||||||
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
|
|||||||
public class MorskaImporter : MorskaBaseImporter
|
public class MorskaImporter : MorskaBaseImporter
|
||||||
{
|
{
|
||||||
public override string ImporterType => "MorskaImporter";
|
public override string ImporterType => "MorskaImporter";
|
||||||
|
|
||||||
private readonly AppDbContext _db;
|
private readonly AppDbContext _db;
|
||||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||||
|
private readonly ILogger<MorskaImporter> _logger;
|
||||||
|
|
||||||
public MorskaImporter(
|
public MorskaImporter(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
SpreadsheetsResource.ValuesResource googleSheetValues)
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||||
|
ILogger<MorskaImporter> logger)
|
||||||
{
|
{
|
||||||
_db = db;
|
_db = db;
|
||||||
_googleSheetValues = googleSheetValues;
|
_googleSheetValues = googleSheetValues;
|
||||||
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
public override void Import(Layer importWorker)
|
public override void Import(Layer importWorker)
|
||||||
{
|
{
|
||||||
|
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||||
|
importWorker.Name, importWorker.Id);
|
||||||
|
|
||||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
||||||
if (sheetId == null)
|
if (sheetId == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
||||||
if (sheetTabName == null)
|
if (sheetTabName == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||||
if (year == null)
|
if (year == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||||
if (month == null)
|
if (month == null)
|
||||||
{
|
{
|
||||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||||
if (name == null)
|
if (name == null)
|
||||||
{
|
{
|
||||||
@@ -52,8 +65,12 @@ public class MorskaImporter : MorskaBaseImporter
|
|||||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||||
|
sheetId, sheetTabName, dataRange);
|
||||||
|
|
||||||
var layer = new Layer
|
var layer = new Layer
|
||||||
{
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
Number = _db.Layers.Count() + 1,
|
Number = _db.Layers.Count() + 1,
|
||||||
ParentId = importWorker.Id,
|
ParentId = importWorker.Id,
|
||||||
Type = LayerType.Import,
|
Type = LayerType.Import,
|
||||||
@@ -66,25 +83,69 @@ public class MorskaImporter : MorskaBaseImporter
|
|||||||
|
|
||||||
var newRecords = new List<Record>();
|
var newRecords = new List<Record>();
|
||||||
|
|
||||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
try
|
||||||
var data = dataRangeResponse.Values;
|
|
||||||
for (var i = 0; i < data[1].Count; i++)
|
|
||||||
{
|
{
|
||||||
if (!(data[0][i].ToString()?.Length > 0) ||
|
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
|
var data = dataRangeResponse.Values;
|
||||||
var record = new Record
|
|
||||||
|
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
||||||
|
|
||||||
|
if (data != null && data.Count >= 2)
|
||||||
{
|
{
|
||||||
Id = Guid.NewGuid(),
|
for (var i = 0; i < data[1].Count; i++)
|
||||||
Code = data[0][i].ToString(),
|
{
|
||||||
Value1 = value,
|
if (!(data[0][i].ToString()?.Length > 0) ||
|
||||||
CreatedAt = DateTime.UtcNow,
|
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||||
ModifiedAt = DateTime.UtcNow
|
{
|
||||||
};
|
_logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
|
||||||
newRecords.Add(record);
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var record = new Record
|
||||||
|
{
|
||||||
|
Id = Guid.NewGuid(),
|
||||||
|
Code = data[0][i].ToString(),
|
||||||
|
Value1 = value,
|
||||||
|
CreatedAt = DateTime.UtcNow,
|
||||||
|
ModifiedAt = DateTime.UtcNow
|
||||||
|
};
|
||||||
|
newRecords.Add(record);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_db.Layers.Add(layer);
|
||||||
|
|
||||||
|
SaveRecords(layer.Id, newRecords);
|
||||||
|
_db.SaveChanges();
|
||||||
|
|
||||||
|
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||||
|
newRecords.Count, layer.Name, layer.Id);
|
||||||
}
|
}
|
||||||
_db.Layers.Add(layer);
|
catch (Exception e)
|
||||||
// TODO: Save records to the layer
|
{
|
||||||
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
|
_logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
|
||||||
_db.SaveChanges();
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||||
|
{
|
||||||
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||||
|
if (toDelete.Count > 0)
|
||||||
|
{
|
||||||
|
_db.Records.RemoveRange(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var record in records)
|
||||||
|
{
|
||||||
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.CreatedAt = DateTime.UtcNow;
|
||||||
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||||
|
record.ModifiedAt = DateTime.UtcNow;
|
||||||
|
record.LayerId = layerId;
|
||||||
|
_db.Records.Add(record);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -321,7 +321,9 @@ public class LayersController : Controller
|
|||||||
.Where(x =>
|
.Where(x =>
|
||||||
x.Name != null && x.Name.Contains(nameFilter) &&
|
x.Name != null && x.Name.Contains(nameFilter) &&
|
||||||
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
||||||
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
|
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True"
|
||||||
|
&& x.Number == 7266
|
||||||
|
)
|
||||||
)
|
)
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
.OrderByDescending(x => x.CreatedAt)
|
||||||
.AsNoTracking()
|
.AsNoTracking()
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
###
|
###
|
||||||
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/D1-
|
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user