mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

imports: save records

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-02 20:11:29 +02:00
parent 5046a29ee1
commit 8efd1edfad
6 changed files with 366 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
View File

@@ -1,5 +1,6 @@
using System.Globalization;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
@@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD1Importer> _logger;
public MorskaD1Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaD1Importer> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
public override void Import(Layer importWorker)
{
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null)
{
@@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}");
}
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
@@ -64,38 +79,74 @@ public class MorskaD1Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
var newRecords = (from t in data
where t.Count > 1 && (string)t[0] != string.Empty
select new Record
{
Id = Guid.NewGuid(),
Code = t[0].ToString(),
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}).ToList();
_db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
try
{
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
var newRecords = (from t in data
where t.Count > 1 && (string)t[0] != string.Empty
select new Record
{
Id = Guid.NewGuid(),
Code = t[0].ToString(),
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}).ToList();
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private double? ParseValue(string? value)
@@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result;
}
catch (FormatException)
catch (FormatException e)
{
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
return null;
}
}
private bool IndexExists(IList<object> array, int index)
{
return array != null && index >= 0 && index < array.Count;

135
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs
View File

@@ -1,6 +1,7 @@
using System.Globalization;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
@@ -9,74 +10,136 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaD3Importer : MorskaBaseImporter
{
public override string ImporterType => "MorskaD3";
private readonly AppDbContext _db;
private readonly ILogger<MorskaD3Importer> _logger;
public MorskaD3Importer(
AppDbContext db)
AppDbContext db,
ILogger<MorskaD3Importer> logger)
{
_db = db;
_logger = logger;
}
public override void Import(Layer importWorker)
{
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null)
{
throw new Exception($"ImportName not found, {importWorker.Name}");
}
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
if (name == null)
if (type == null)
{
throw new Exception($"ImportType not found, {importWorker.Name}");
}
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
if (dataInbox == null)
{
throw new Exception($"DataInbox not found, {type}");
}
var data = Convert.FromBase64String(dataInbox.Data);
var tst = Encoding.UTF8.GetString(data);
var records = JsonSerializer.Deserialize<List<Record>>(Encoding.UTF8.GetString(data));
if (records == null)
{
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
}
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
if (records.Count == 0)
{
throw new Exception($"No records found for {year}{month}");
}
records = records.Select(x =>
{
x.Id = Guid.NewGuid();
x.CreatedAt = DateTime.UtcNow;
x.ModifiedAt = DateTime.UtcNow;
return x;
}).ToList();
var layer = new Layer
{
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
_db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, records, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
_logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
dataInbox.Id, dataInbox.CreatedAt);
try
{
var data = Convert.FromBase64String(dataInbox.Data);
var jsonString = Encoding.UTF8.GetString(data);
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
if (records == null)
{
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
}
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
if (records.Count == 0)
{
throw new Exception($"No records found for {year}{month}");
}
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
records.Count, year, month);
records = records.Select(x =>
{
x.Id = Guid.NewGuid();
x.CreatedAt = DateTime.UtcNow;
x.ModifiedAt = DateTime.UtcNow;
return x;
}).ToList();
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
_db.Layers.Add(layer);
SaveRecords(layer.Id, records);
_db.SaveChanges();
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
records.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

126
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
View File

@@ -1,5 +1,6 @@
using System.Globalization;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaFk2Importer : MorskaBaseImporter
{
public override string ImporterType => "MorskaFK2";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaFk2Importer> _logger;
public MorskaFk2Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaFk2Importer> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
_logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null)
{
@@ -52,8 +65,12 @@ public class MorskaFk2Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}");
}
_logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
@@ -62,40 +79,91 @@ public class MorskaFk2Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (var i = 0; i < data.Count; i++)
try
{
if (data[i].Count <= 9 || (string)data[i][3] == string.Empty) continue;
var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3)
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null)
{
throw new Exception($"Invalid date in row {i}");
for (var i = 0; i < data.Count; i++)
{
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i);
continue;
}
var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3)
{
_logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]);
throw new Exception($"Invalid date in row {i}");
}
var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i);
continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][3].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
}
var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (!(data[i][9].ToString()?.Length > 0) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
var record = new Record
{
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][3].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
_db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
catch (Exception e)
{
_logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

97
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs
View File

@@ -1,5 +1,6 @@
using System.Globalization;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaImporter : MorskaBaseImporter
{
public override string ImporterType => "MorskaImporter";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaImporter> _logger;
public MorskaImporter(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaImporter> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null)
{
@@ -52,8 +65,12 @@ public class MorskaImporter : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}");
}
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
@@ -66,25 +83,69 @@ public class MorskaImporter : MorskaBaseImporter
var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (var i = 0; i < data[1].Count; i++)
try
{
if (!(data[0][i].ToString()?.Length > 0) ||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
var record = new Record
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null && data.Count >= 2)
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
for (var i = 0; i < data[1].Count; i++)
{
if (!(data[0][i].ToString()?.Length > 0) ||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
}
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
_db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
catch (Exception e)
{
_logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

4
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -321,7 +321,9 @@ public class LayersController : Controller
.Where(x =>
x.Name != null && x.Name.Contains(nameFilter) &&
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True"
&& x.Number == 7266
)
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()

2
tools/http-tests/AutoImport.http
View File

@@ -1,3 +1,3 @@
###
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/D1-
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-