imports: save records

2025-06-02 20:11:29 +02:00
parent 5046a29ee1
commit 8efd1edfad
6 changed files with 366 additions and 119 deletions
--- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
+++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
@@ -1,5 +1,6 @@
 using System.Globalization;
 using Google.Apis.Sheets.v4;
+using Microsoft.Extensions.Logging;
 using DiunaBI.Core.Models;
 using DiunaBI.Database.Context;

@@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter

    private readonly AppDbContext _db;
    private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
+    private readonly ILogger<MorskaD1Importer> _logger;

    public MorskaD1Importer(
        AppDbContext db,
-        SpreadsheetsResource.ValuesResource googleSheetValues)
+        SpreadsheetsResource.ValuesResource googleSheetValues,
+        ILogger<MorskaD1Importer> logger)
    {
        _db = db;
        _googleSheetValues = googleSheetValues;
+        _logger = logger;
    }

-    public override void  Import(Layer importWorker)
+    public override void Import(Layer importWorker)
    {
+        _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", 
+            importWorker.Name, importWorker.Id);
+
        var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
        if (sheetId == null)
        {
            throw new Exception($"SheetId not found, {importWorker.Name}");
        }
+
        var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
        if (sheetTabName == null)
        {
            throw new Exception($"SheetTabName not found, {importWorker.Name}");
        }
+
        var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
        if (year == null)
        {
            throw new Exception($"ImportYear not found, {importWorker.Name}");
        }
+
        var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
        if (month == null)
        {
            throw new Exception($"ImportMonth not found, {importWorker.Name}");
        }
+
        var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
        if (name == null)
        {
@@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter
            throw new Exception($"DataRange not found, {importWorker.Name}");
        }

+        _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
+            sheetId, sheetTabName, dataRange);
+
        var layer = new Layer
        {
+            Id = Guid.NewGuid(), 
            Number = _db.Layers.Count() + 1,
            ParentId = importWorker.Id,
            Type = LayerType.Import,
@@ -64,38 +79,74 @@ public class MorskaD1Importer : MorskaBaseImporter
            CreatedAt = DateTime.UtcNow,
            ModifiedAt = DateTime.UtcNow
        };
-        layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
+        layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";

-        var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
-        var data = dataRangeResponse.Values;
-        var newRecords = (from t in data
-                        where t.Count > 1 && (string)t[0] != string.Empty
-                        select new Record
-                        {
-                            Id = Guid.NewGuid(),
-                            Code = t[0].ToString(),
-                            Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
-                            Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
-                            Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
-                            Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
-                            Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
-                            Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
-                            Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
-                            Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
-                            Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
-                            Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
-                            Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
-                            Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
-                            Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
-                            Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
-                            Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
-                            CreatedAt = DateTime.UtcNow,
-                            ModifiedAt = DateTime.UtcNow
-                        }).ToList();
-        _db.Layers.Add(layer);
-        // TODO: Save records to the layer
-        //controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
-        _db.SaveChanges();
+        try
+        {
+            var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
+            var data = dataRangeResponse.Values;
+
+            _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
+
+            var newRecords = (from t in data
+                            where t.Count > 1 && (string)t[0] != string.Empty
+                            select new Record
+                            {
+                                Id = Guid.NewGuid(),
+                                Code = t[0].ToString(),
+                                Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
+                                Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
+                                Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
+                                Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
+                                Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
+                                Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
+                                Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
+                                Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
+                                Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
+                                Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
+                                Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
+                                Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
+                                Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
+                                Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
+                                Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
+                                CreatedAt = DateTime.UtcNow,
+                                ModifiedAt = DateTime.UtcNow
+                            }).ToList();
+
+            _db.Layers.Add(layer);
+            
+            SaveRecords(layer.Id, newRecords);
+            _db.SaveChanges();
+
+            _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
+                newRecords.Count, layer.Name, layer.Id);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
+            throw;
+        }
+    }
+
+    private void SaveRecords(Guid layerId, ICollection<Record> records)
+    {
+        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
+        if (toDelete.Count > 0)
+        {
+            _db.Records.RemoveRange(toDelete);
+        }
+
+        foreach (var record in records)
+        {
+            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            record.CreatedAt = DateTime.UtcNow;
+            record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            record.ModifiedAt = DateTime.UtcNow;
+            record.LayerId = layerId;
+            _db.Records.Add(record);
+        }
+
+        _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
    }

    private double? ParseValue(string? value)
@@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter
            double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
            return result;
        }
-        catch (FormatException)
+        catch (FormatException e)
        {
+            _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
            return null;
        }
    }
+
    private bool IndexExists(IList<object> array, int index)
    {
        return array != null && index >= 0 && index < array.Count;