Import data from GoogleSheet

Frontend/src/app/models/dataSet.model.ts

@@ -29,6 +29,7 @@ export class DataSet extends Base {
             id: [null],
             name: ['', Validators.required],
             source: ['', Validators.required],
+            sheetId: '1G_Hu8DTP-PSPNXTaVYhc_ppnTQi6HWoA4oXSSdUmM9E',
             createdAt: '',
             modifiedAt: '',
             createdBy: '',
@@ -91,4 +92,16 @@ export class DataSet extends Base {
             })
         })
     }
+    static parseGoogleSheet(sheetId: string, _http: HttpClient): Promise<DataRow[]> {
+        return new Promise((resolve, reject) => {
+            _http.get<DataRow[]>(`${environment.api.url}/datasets/parseGoogleSheet/${sheetId}`,
+            ).pipe(map(data  => data.map(x => new DataRow().deserialize(x))))
+            .subscribe({
+                next: (data) => {
+                    resolve(data);
+                },
+                error: (e) => reject(e)
+            })
+        })
+    }
 }

Frontend/src/app/modules/data-sets/data-set-edit/data-set-edit.component.html

@@ -1,69 +1,82 @@
 <div>
-    <form [formGroup]="form" (ngSubmit)="save()" novalidate>
-      <mat-card appearance="outlined">
-        <mat-toolbar color="secondary">
-          Edycja warstwy danych
-          <span class="fill-to-right"></span>
-          <button mat-button type="submit" [disabled]="form.invalid">Zapisz</button>
-          <button mat-button type="button" routerLink="../..">Anuluj</button>
-        </mat-toolbar>
-        <mat-card-content>
-          <mat-grid-list cols="2" rowHeight="90px">
-            <mat-grid-tile>
-              <mat-form-field class="detail-input">
-                <mat-label>Nazwa</mat-label>
-                <input matInput formControlName="name">
-                <mat-error *ngIf="form.controls['name'].touched && form.controls['name'].invalid">
-                  Pole obowiązkowe
-                </mat-error>
-              </mat-form-field>
-            </mat-grid-tile>
-  
-            <mat-grid-tile>
-                <mat-form-field class="detail-input">
-                  <mat-select placeholder="Źródło" formControlName="source" (selectionChange)="generateNumber()">
-                    <mat-option value="CSV">CSV</mat-option>
-                    <mat-option value="GoogleSheet">GoogleSheet</mat-option>
-                    <mat-option value="SAP">SAP</mat-option>
-                    <mat-option value="Symfonia">Symfonia</mat-option>
-                  </mat-select>
-                  <mat-error *ngIf="form.controls['source'].touched && form.controls['source'].invalid">
-                    Pole obowiązkowe
-                  </mat-error>
-                </mat-form-field>
-              </mat-grid-tile>
-          </mat-grid-list>
+  <form [formGroup]="form" (ngSubmit)="save()" novalidate>
+    <mat-card appearance="outlined">
+      <mat-toolbar color="secondary">
+        Edycja warstwy danych
+        <span class="fill-to-right"></span>
+        <button mat-button type="submit" [disabled]="form.invalid">Zapisz</button>
+        <button mat-button type="button" routerLink="../..">Anuluj</button>
+      </mat-toolbar>
+      <mat-card-content>
+        <mat-grid-list cols="2" rowHeight="90px">
+          <mat-grid-tile>
+            <mat-form-field class="detail-input">
+              <mat-label>Nazwa</mat-label>
+              <input matInput formControlName="name">
+              <mat-error *ngIf="form.controls['name'].touched && form.controls['name'].invalid">
+                Pole obowiązkowe
+              </mat-error>
+            </mat-form-field>
+          </mat-grid-tile>
 
-          <input type="file" class="file-input" (change)="onFileSelected($event)" #fileUpload>
-          <button mat-mini-fab color="primary" type="button" class="upload-btn" (click)="fileUpload.click()">
-            <mat-icon>attach_file</mat-icon>
-          </button>
+          <mat-grid-tile>
+            <mat-form-field class="detail-input">
+              <mat-select placeholder="Źródło" formControlName="source" (selectionChange)="generateNumber()">
+                <mat-option value="CSV">CSV</mat-option>
+                <mat-option value="GoogleSheet">GoogleSheet</mat-option>
+                <mat-option value="SAP">SAP</mat-option>
+                <mat-option value="Symfonia">Symfonia</mat-option>
+              </mat-select>
+              <mat-error *ngIf="form.controls['source'].touched && form.controls['source'].invalid">
+                Pole obowiązkowe
+              </mat-error>
+            </mat-form-field>
+          </mat-grid-tile>
+        </mat-grid-list>
+        <input type="file" class="file-input" (change)="onFileSelected($event)" #fileUpload>
+        <button mat-mini-fab color="primary" type="button" class="upload-btn" (click)="fileUpload.click()"
+          *ngIf="form.get('source')?.value === 'CSV'">
+          <mat-icon>attach_file</mat-icon>
+        </button>
+        <mat-grid-list cols="5" rowHeight="90px" *ngIf="form.get('source')?.value === 'GoogleSheet'">
+          <mat-grid-tile>
+            <mat-form-field class>
+              <input matInput formControlName="sheetId">
+            </mat-form-field>
+          </mat-grid-tile>
+          <mat-grid-tile>
+            <button mat-mini-fab color="primary" type="button" class="upload-btn" (click)="parseGoogleSheet()">
+              <mat-icon>publish</mat-icon>
+            </button>
+          </mat-grid-tile>
+          <mat-grid-tile></mat-grid-tile>
+          <mat-grid-tile></mat-grid-tile>
+          <mat-grid-tile></mat-grid-tile>
+        </mat-grid-list>
 
-          <mat-table #table [dataSource]="dataSource" [trackBy]="trackByUid" matSort class="animate">
-  
-            <ng-container matColumnDef="code">
-              <mat-header-cell *matHeaderCellDef mat-sort-header>MPK</mat-header-cell>
-              <mat-cell *matCellDef="let item">{{item.code}}</mat-cell>
-            </ng-container>
-      
-            <ng-container matColumnDef="desc1">
-              <mat-header-cell *matHeaderCellDef mat-sort-header>Konto</mat-header-cell>
-              <mat-cell *matCellDef="let item">{{item.desc1}}</mat-cell>
-            </ng-container>
+        <mat-table #table [dataSource]="dataSource" [trackBy]="trackByUid" matSort class="animate">
 
-            <ng-container matColumnDef="value">
-              <mat-header-cell *matHeaderCellDef mat-sort-header>Wartość</mat-header-cell>
-              <mat-cell *matCellDef="let item">{{item.value | number:'1.2-2'}}</mat-cell>
-            </ng-container>
-        
-            <mat-header-row *matHeaderRowDef="displayedColumns"></mat-header-row>
-            <mat-row *matRowDef="let item; columns: displayedColumns;"></mat-row>
-          </mat-table>
-          <mat-paginator #paginator
-                         [pageSize]="50"
-                         [pageSizeOptions]="[5, 10, 20]">
-          </mat-paginator>
-        </mat-card-content>
-      </mat-card>
-    </form>
-  </div>
+          <ng-container matColumnDef="code">
+            <mat-header-cell *matHeaderCellDef mat-sort-header>MPK</mat-header-cell>
+            <mat-cell *matCellDef="let item">{{item.code}}</mat-cell>
+          </ng-container>
+
+          <ng-container matColumnDef="desc1">
+            <mat-header-cell *matHeaderCellDef mat-sort-header>Konto</mat-header-cell>
+            <mat-cell *matCellDef="let item">{{item.desc1}}</mat-cell>
+          </ng-container>
+
+          <ng-container matColumnDef="value">
+            <mat-header-cell *matHeaderCellDef mat-sort-header>Wartość</mat-header-cell>
+            <mat-cell *matCellDef="let item">{{item.value | number:'1.2-2'}}</mat-cell>
+          </ng-container>
+
+          <mat-header-row *matHeaderRowDef="displayedColumns"></mat-header-row>
+          <mat-row *matRowDef="let item; columns: displayedColumns;"></mat-row>
+        </mat-table>
+        <mat-paginator #paginator [pageSize]="50" [pageSizeOptions]="[5, 10, 20]">
+        </mat-paginator>
+      </mat-card-content>
+    </mat-card>
+  </form>
+</div>

Frontend/src/app/modules/data-sets/data-set-edit/data-set-edit.component.ts

@@ -68,4 +68,11 @@ export class DataSetEditComponent implements OnInit {
   trackByUid(index: number, item: DataRow) {
     return item.id;
   }
+  async parseGoogleSheet() {
+    const id = this.form.get('sheetId')?.value;
+    this.document.dataRows = await DataSet.parseGoogleSheet(id, this.http$);
+    this.dataSource = new MatTableDataSource(this.document.dataRows);
+    this.dataSource.paginator = this.paginator;
+    this.dataSource.sort = this.sort;
+  }
 }

WebAPI/Controllers/DataSetsController.cs

@@ -1,5 +1,6 @@
 using Google.Apis.Auth;
 using Google.Apis.Http;
+using Google.Apis.Sheets.v4;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
@@ -17,11 +18,15 @@ namespace WebAPI.Controllers
 {
     [ApiController]
     [Route("api/[controller]")]
-    [Authorize]
+    // [Authorize]
     public class DataSetsController : Controller
     {
         private readonly AppDbContext db;
-        public DataSetsController(AppDbContext _db) { db = _db; }
+        private SpreadsheetsResource.ValuesResource googleSheetValues;
+        public DataSetsController(AppDbContext _db, GoogleSheetsHelper _googleSheetsHelper) { 
+            db = _db;
+            googleSheetValues = _googleSheetsHelper.Service.Spreadsheets.Values;
+        }
 
         [HttpGet]
         public IActionResult GetAll()
@@ -64,6 +69,14 @@ namespace WebAPI.Controllers
                 return BadRequest(e.ToString());
             }
         }
+        [HttpGet]
+        [Route("parseGoogleSheet/{sheetId}")]
+        public IActionResult ParseGoogleSheet(string sheetId)
+        {
+
+            var parser = new googleSheetParser(googleSheetValues);
+            return Ok(parser.parse(sheetId));
+        }
         [HttpPost]
         [DisableRequestSizeLimit]
         [Route("parseFile")]

WebAPI/GoogleSheetsHelper.cs

@@ -0,0 +1,35 @@
+using Google.Apis.Auth.OAuth2;
+using Google.Apis.Services;
+using Google.Apis.Sheets.v4;
+
+namespace WebAPI
+{
+    public class GoogleSheetsHelper
+    {
+        public SheetsService Service { get; set; }
+        const string APPLICATION_NAME = "Diuna";
+        static readonly string[] Scopes = { SheetsService.Scope.Spreadsheets };
+        public GoogleSheetsHelper()
+        {
+            InitializeService();
+        }
+        private void InitializeService()
+        {
+            var credential = GetCredentialsFromFile();
+            Service = new SheetsService(new BaseClientService.Initializer()
+            {
+                HttpClientInitializer = credential,
+                ApplicationName = APPLICATION_NAME
+            });
+        }
+        private GoogleCredential GetCredentialsFromFile()
+        {
+            GoogleCredential credential;
+            using (var stream = new FileStream("client_secrets.json", FileMode.Open, FileAccess.Read))
+            {
+                credential = GoogleCredential.FromStream(stream).CreateScoped(Scopes);
+            }
+            return credential;
+        }
+    }
+}

WebAPI/Program.cs

@@ -53,6 +53,8 @@ builder.Services.AddAuthorization();
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen();
 
+builder.Services.AddSingleton(typeof(GoogleSheetsHelper));
+
 var app = builder.Build();
 
 app.Use(async (context, next) =>

WebAPI/WebAPI.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <PackageReference Include="CsvHelper" Version="30.0.1" />
     <PackageReference Include="Google.Apis.Auth" Version="1.58.0" />
+    <PackageReference Include="Google.Apis.Sheets.v4" Version="1.58.0.2826" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="7.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="7.0.0" />

WebAPI/client_secrets.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "diuna-370117",
+  "private_key_id": "f48fd588724e6733b9639fe7d7933091b96be34f",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCenqveXpGKXA10\npsAQ4Wreeiom9GMbZywnqMAhxc0wobI7EfnbP4FPOjfS8oWFRRrVzRil78zeUGWX\nb1WMHYvUyU3IrGXp6kVxuxbjBvwooOB5cEgz928A3aUUZRXxwjPV3+KuuAeQydVw\nPMQo2a0AQ+YAOK2QMG+BGAPAzYB+/35Zf6JsDOIDgWMaJq3etKgIijk40Nmf+uaG\nRAQlEbMhnAaAYz2B6I7W3z0pFDq2btgYJII+DWRC2DjSrA4UUeuds8Kz5qwfafJ8\nki9N1RdYdbB/q6T74xQ3G/aEOK+CYmkWQz2woY5y8b5RCbKoGGIXpu6FVuWTnVxY\nJpP5QvIFAgMBAAECggEAJC3Evb+MKqa8WvL9s9v2aDAtFR2AzWtG4vTWfd2D46e9\n40NCXgOqFswMl4zBb5hHeqSBDrgXXk2wHk5CkObcUfhoSXEo/aV1mW821SluskWf\nbZNypIe3RddII9K6op3M/OdH6NoIv7mJeUQi6b5ce0cBWuOSkuS5ShSUJpG40T5R\nQfl0iMuEYDpU1tvKmwhFlPTUTUGH7RdeqGFYIfE3kzFQiiSrS8V5L1GJKWcxMLdT\nq4P9JzaSW7eAAYKJiFTMSQvqs7pssCIj1JNLzD9PTsQmid2V2mUJIg3joXMNGbxN\nqMcIqbEesidIsDOkQ06taUIYG39og6rc9bar6XWRgQKBgQDK/+a8jCmUByhedUT5\nZnREtHm4HcVo1tfBcmmqSEV0VJPJd14+CYvaUzCCJ9+xiLo6yOWRUk2h1GANAp50\nAdiVAHNibfwtri7vKWNhpnd111N/ebh6GIksT0ZTvu7sq5qbYXU3q6l6YRCyXSdF\n1oRfQED8I8G1xZP5j6fspBgoKQKBgQDICIKo3gmUEeFSt+o+Lucd2BljaFq/hUMA\n6WFdKbRyyd2iKBmGR15VNihiuJWy5i2nmuFaXMkeHo/PUJeEYC+vkc7M7UCYtD9l\n2xwp78o3ss7vxdPvOKhrcvux/Wpk1nuAEpM459MC0bmtOGIKU+QmDbsBbMHZ6p0R\n8DvECJ9mfQKBgEj60PAOD9CY9ilnTYHAFKKyo2POyC7VtkFkqZo/W0DkOzFdybLR\n6cZ2y+SvAxunRRRnLykchq5cVJ+4xlB8bWm7/L9xPQ0LJvJyVblAiIgD/o/AqdKz\nSXV1lpn69Zh+ZRnhYqu9+jL1/HOzS7Au2+4GgpZjIHwB6R36SGup3slpAoGAZW2j\nSxsjQjh6x2XIWfWQbVqZLQXKFhjta7XrD8FI5XekcUfiAWuI0q5edghgp9D9T2JC\naH5p4GLgyt9zpMTdCSpm8RRQT93905jxw/X51JpPQddO6psRE0K/i3YTD8SN5NgG\nXLF4FpLfkozncZMuOXl23HcYKHZFZMYql/FDWkUCgYAjGQKzYV7IXA7UDAY3ejaw\nWMbsDttSPQ0E1ouuJWIX/eb4SXYr0u/gdLuX1uM7EsxqIGVFWfgtUGopoVGr604S\ng+dfOPZgUzaGAlUE2iRMVp6YoRRbrvPsYJwDrV0Xwil1k6UEzn8bgXO/IQ4fgIWj\nkxS5sDkZ6LVSCfDn5tLThg==\n-----END PRIVATE KEY-----\n",
+  "client_email": "diuna-backend@diuna-370117.iam.gserviceaccount.com",
+  "client_id": "101546901561736131820",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/diuna-backend%40diuna-370117.iam.gserviceaccount.com"
+}

WebAPI/dataParsers/googleSheet.parser.cs

@@ -0,0 +1,43 @@
+using Google.Apis.Sheets.v4;
+using System.Globalization;
+using WebAPI.Models;
+
+namespace WebAPI.dataParsers
+{
+    public class googleSheetParser
+    {
+        private SpreadsheetsResource.ValuesResource googleSheetValues;
+
+        public googleSheetParser(SpreadsheetsResource.ValuesResource _googleSheetValues)
+        {
+            googleSheetValues = _googleSheetValues;
+        }
+
+        public List<DataRow> parse(string sheetId)
+        {
+            var range = "Arkusz1!A:B";
+
+            var request = googleSheetValues.Get(sheetId, range);
+            var response = request.Execute();
+            var data = response.Values;
+
+            List<DataRow> dataRows = new List<DataRow>();
+
+            for (int i = 1; i < data.Count; i++)
+            {
+                float value = float.Parse(data[i][1].ToString(), CultureInfo.GetCultureInfo("pl-PL"));
+                if (value > 0)
+                {
+                    DataRow dataRow = new DataRow();
+                    dataRow.Id = Guid.NewGuid();
+                    dataRow.Code = data[i][0].ToString();
+                    dataRow.Value = value;
+                    dataRow.CreatedAt = DateTime.UtcNow;
+                    dataRow.ModifiedAt = DateTime.UtcNow;
+                    dataRows.Add(dataRow);
+                }
+            }
+            return dataRows;
+        }
+    }
+}