[Feature/Perf] Ottimizzazioni bulk pre-discovery, batch deletion sync e supporto OLE DB / Salesforce client_credentials

## Bulk Pre-Discovery e riduzione query SQLite/SOQL ### KeyAssociationService — FindAssociationsByKeyValuesBulkAsync (nuovo) - Aggiunta query bulk 'WHERE KeyValue IN (...)' per recuperare N associazioni con 1 sola query SQLite (chunking a 500 chiavi per rispettare il limite ~999 parametri di SQLite) - Aggiunta interfaccia IKeyAssociationService e delegata in DataConnectionCredentialService / IDataConnectionCredentialService ### AssociationService — BatchFindOrCreateAssociationsAsync (nuovo) - Nuovo metodo bulk che sostituisce i loop per-record durante l'analisi composite: 1) 1 query SQLite bulk per tutte le chiavi 2) Per le chiavi non trovate: SOQL 'IN (...)' su Salesforce in chunk da 200 via BatchExecuteQueriesAsync (ceil(K/25) HTTP Composite call invece di K singole) 3) Salvataggio parallelo delle associazioni pre-discovery scoperte - Fallback per-record automatico per client REST non Salesforce - Aggiornata interfaccia IAssociationService con documentazione XML completa ### DataCoupler.razor.cs — STEP A/B nel flusso COMPOSITE - Pre-Discovery spostata FUORI dal loop parallelo (STEP A, prima dell'analisi) - associationsByKey pre-popolato con BatchFindOrCreateAssociationsAsync - STEP B: il loop parallelo usa TryGetValue O(1) invece di query async per record - Rimozione blocco ~40 righe di per-record lookup / fallback duplicati ## Salesforce Composite API — Batch Delete e Patch ### SalesforceServiceClient — metodi batch (nuovi) - BatchDeleteEntitiesAsync: elimina N record con ceil(N/25) Composite call invece di N - BatchPatchSingleFieldAsync: aggiorna un singolo campo su N record tramite BatchUpdateEntitiesAsync ### DeletionSyncService — refactoring batch - ExecuteBatchedSalesforceDeletionsAsync: orchestrazione batch per Delete / Deactivate / Mark su Salesforce - ExecuteSequentialDeletionsAsync: loop sequenziale esistente estratto in metodo riutilizzabile - Dispatcher: Salesforce -> batch Composite, altri client REST -> sequenziale ## Supporto OLE DB (database) ### DatabaseSchemaProviderFactory - Aggiunto case DatabaseType.OleDb -> new OleDbSchemaProvider() nel factory switch ### DatabaseMethod.cs - Aggiunto metodo IsOleDbConnection() (parallelo a IsOdbcConnection()) - Query validation e manager temporaneo estesi a OLE DB oltre che ODBC - GetLimitedQuery: aggiunto case OleDb -> 'SELECT TOP N FROM (subquery)' ## Salesforce OAuth2 — fix client_credentials ### CredentialService.cs - Aggiunto 'GrantType' alla HashSet serviceSpecificKeys per preservarlo nella serializzazione AdditionalParameters ### DataConnectionCredentialService.cs - Refactored BuildRestServiceOptions in helper statico riutilizzato da entrambi i metodi GetRestServiceOptions - Mapping coerente ClientId/ClientSecret/GrantType per Salesforce (allineato a DataConnectionFactory) - TestSalesforceOAuthLogin: branch esplicito per client_credentials (no username/password/token) con validazione preventiva ClientId+ClientSecret obbligatori - Log flow label (password|client_credentials) in tutti i messaggi di autenticazione ## VS Code tasks ### .vscode/tasks.json - Rimosso task generico 'Publish Data_Coupler' - Aggiunti due task separati: win-x64 e win-x86, entrambi SingleFile + Self-Contained + ReadyToRun
2026-05-28 11:15:18 +02:00
parent 82e0d6bc77
commit 344853fde9
13 changed files with 886 additions and 223 deletions
@@ -1,11 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
 using CredentialManager.Models;
 using CredentialManager.Services;
 using DataConnection.CredentialManagement.Interfaces;
+using DataConnection.REST.Implementations;
 using DataConnection.REST.Interfaces;
 using Microsoft.Extensions.Logging;

@@ -69,6 +71,227 @@ public class AssociationService : IAssociationService
        return null;
    }

+    /// <summary>
+    /// Versione bulk del find-or-create — vedi <see cref="IAssociationService.BatchFindOrCreateAssociationsAsync"/>.
+    /// </summary>
+    public async Task<Dictionary<string, KeyAssociation>> BatchFindOrCreateAssociationsAsync(
+        IEnumerable<string> sourceKeys,
+        PreDiscoveryRequest commonRequest)
+    {
+        if (commonRequest == null)
+            throw new ArgumentNullException(nameof(commonRequest));
+
+        var distinctKeys = sourceKeys
+            .Where(k => !string.IsNullOrEmpty(k))
+            .Distinct()
+            .ToList();
+
+        var result = new Dictionary<string, KeyAssociation>(StringComparer.Ordinal);
+        if (distinctKeys.Count == 0)
+            return result;
+
+        // STEP 1 — Bulk lookup nel DB locale (1 query SQLite invece di N)
+        Dictionary<string, KeyAssociation> localMatches;
+        try
+        {
+            localMatches = await _credentialService.FindKeyAssociationsByValuesBulkAsync(
+                distinctKeys, commonRequest.DestinationEntity, commonRequest.CredentialName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "BULK PRE-DISCOVERY: bulk lookup locale fallito, fallback per-record");
+            // Fallback per-record (mantiene comportamento esistente in caso di problemi)
+            foreach (var key in distinctKeys)
+            {
+                var req = ClonePreDiscoveryRequest(commonRequest, key);
+                var found = await FindOrCreateAssociationAsync(req);
+                if (found != null) result[key] = found;
+            }
+            return result;
+        }
+
+        foreach (var kvp in localMatches)
+            result[kvp.Key] = kvp.Value;
+
+        var missingKeys = distinctKeys.Where(k => !result.ContainsKey(k)).ToList();
+
+        _logger.LogInformation("BULK PRE-DISCOVERY: {Local}/{Total} associazioni trovate localmente, {Missing} da cercare nella destinazione",
+            localMatches.Count, distinctKeys.Count, missingKeys.Count);
+
+        if (missingKeys.Count == 0 || !commonRequest.EnablePreDiscovery)
+            return result;
+
+        // STEP 2 — Pre-Discovery batched sulla destinazione REST
+        // Verifica che il campo chiave sia mappato
+        if (string.IsNullOrEmpty(commonRequest.SourceKeyField) ||
+            !commonRequest.FieldMappings.TryGetValue(commonRequest.SourceKeyField, out var mappedField))
+        {
+            _logger.LogWarning("BULK PRE-DISCOVERY: campo chiave '{SourceKeyField}' non mappato; skip discovery",
+                commonRequest.SourceKeyField);
+            return result;
+        }
+
+        // Solo SalesforceServiceClient supporta SOQL IN ottimizzata; per altri client si ricade al per-record.
+        if (commonRequest.RestClient is SalesforceServiceClient sfClient)
+        {
+            var discovered = await PerformBulkPreDiscoverySalesforceAsync(
+                sfClient, missingKeys, mappedField, commonRequest);
+
+            foreach (var kvp in discovered)
+                result[kvp.Key] = kvp.Value;
+        }
+        else
+        {
+            _logger.LogDebug("BULK PRE-DISCOVERY: client REST non Salesforce, fallback per-record per {Count} chiavi", missingKeys.Count);
+            foreach (var key in missingKeys)
+            {
+                var req = ClonePreDiscoveryRequest(commonRequest, key);
+                var found = await PerformPreDiscoveryAsync(req);
+                if (found != null) result[key] = found;
+            }
+        }
+
+        return result;
+    }
+
+    /// <summary>
+    /// Pre-Discovery batched specifica per Salesforce: usa SOQL <c>WHERE field IN (...)</c>
+    /// per recuperare in pochissime chiamate API tutti i record che matchano una qualsiasi delle chiavi mancanti.
+    /// </summary>
+    private async Task<Dictionary<string, KeyAssociation>> PerformBulkPreDiscoverySalesforceAsync(
+        SalesforceServiceClient sfClient,
+        List<string> missingKeys,
+        string mappedDestinationField,
+        PreDiscoveryRequest commonRequest)
+    {
+        var output = new Dictionary<string, KeyAssociation>(StringComparer.Ordinal);
+
+        // Chunk per stare sotto il limite SOQL/URL (~16 KB GET): ~200 valori per query
+        const int chunkSize = 200;
+        var queries = new List<string>();
+        for (int i = 0; i < missingKeys.Count; i += chunkSize)
+        {
+            var chunk = missingKeys.Skip(i).Take(chunkSize).ToList();
+
+            var sb = new StringBuilder();
+            sb.Append("SELECT Id, ");
+            sb.Append(mappedDestinationField);
+            sb.Append(" FROM ");
+            sb.Append(commonRequest.DestinationEntity);
+            sb.Append(" WHERE ");
+            sb.Append(mappedDestinationField);
+            sb.Append(" IN (");
+            sb.Append(string.Join(",", chunk.Select(v => $"'{v.Replace("'", "\\'")}'")));
+            sb.Append(')');
+
+            queries.Add(sb.ToString());
+        }
+
+        _logger.LogInformation("BULK PRE-DISCOVERY: {QueryCount} SOQL IN-query (~{ChunkSize} chiavi/query, Composite API ceil(N/25) HTTP call)",
+            queries.Count, chunkSize);
+
+        // BatchExecuteQueriesAsync raggruppa fino a 25 query in 1 Composite request
+        var batchResults = await sfClient.BatchExecuteQueriesAsync(queries);
+
+        // Indicizza i risultati per chiave: dal record letto leggiamo il valore di mappedDestinationField
+        var entityIdByKey = new Dictionary<string, string>(StringComparer.Ordinal);
+        foreach (var batchResult in batchResults.Where(r => r.Success && r.Records != null))
+        {
+            foreach (var record in batchResult.Records)
+            {
+                if (!record.TryGetValue(mappedDestinationField, out var keyVal) || keyVal == null)
+                    continue;
+
+                var keyStr = keyVal.ToString();
+                if (string.IsNullOrEmpty(keyStr))
+                    continue;
+
+                var idStr = ExtractDestinationId(record);
+                if (string.IsNullOrEmpty(idStr))
+                    continue;
+
+                // In caso di duplicati in Salesforce, prendiamo il primo
+                if (!entityIdByKey.ContainsKey(keyStr))
+                    entityIdByKey[keyStr] = idStr;
+            }
+        }
+
+        _logger.LogInformation("BULK PRE-DISCOVERY: trovati {Found}/{Missing} record esistenti nella destinazione",
+            entityIdByKey.Count, missingKeys.Count);
+
+        if (entityIdByKey.Count == 0)
+            return output;
+
+        // Salvataggio associazioni Pre-Discovery in parallelo
+        var saveTasks = entityIdByKey.Select(async kvp =>
+        {
+            try
+            {
+                var newAssoc = new KeyAssociation
+                {
+                    KeyValue = kvp.Key,
+                    SourceKeyField = commonRequest.SourceKeyField,
+                    DestinationKeyField = commonRequest.DestinationKeyField ?? "Id",
+                    MappedDestinationField = mappedDestinationField,
+                    DestinationEntity = commonRequest.DestinationEntity,
+                    DestinationId = kvp.Value,
+                    RestCredentialName = commonRequest.CredentialName,
+                    CreatedAt = DateTime.UtcNow,
+                    LastVerifiedAt = DateTime.UtcNow,
+                    IsActive = true,
+                    AdditionalInfo = JsonSerializer.Serialize(new Dictionary<string, object>
+                    {
+                        { "CreatedBy", "PreDiscovery" },
+                        { "DiscoveredAt", DateTime.UtcNow },
+                        { "MappingCount", commonRequest.FieldMappings.Count },
+                        { "BulkPreDiscovery", true },
+                        { "ScheduledTransfer", commonRequest.IsScheduledTransfer },
+                        { "SourceType", commonRequest.SourceType ?? string.Empty }
+                    })
+                };
+
+                var id = commonRequest.UseParallelMethod
+                    ? await _credentialService.SaveKeyAssociationParallelAsync(newAssoc)
+                    : await _credentialService.SaveKeyAssociationAsync(newAssoc);
+
+                newAssoc.Id = id;
+                return (kvp.Key, newAssoc);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "BULK PRE-DISCOVERY: errore nel salvataggio associazione per KeyValue '{KeyValue}'", kvp.Key);
+                return (kvp.Key, (KeyAssociation?)null);
+            }
+        });
+
+        var savedResults = await Task.WhenAll(saveTasks);
+        foreach (var (key, assoc) in savedResults)
+        {
+            if (assoc != null) output[key] = assoc;
+        }
+
+        return output;
+    }
+
+    private static PreDiscoveryRequest ClonePreDiscoveryRequest(PreDiscoveryRequest source, string sourceKey)
+    {
+        return new PreDiscoveryRequest
+        {
+            SourceKey = sourceKey,
+            SourceKeyField = source.SourceKeyField,
+            DestinationEntity = source.DestinationEntity,
+            CredentialName = source.CredentialName,
+            DestinationKeyField = source.DestinationKeyField,
+            FieldMappings = source.FieldMappings,
+            RestClient = source.RestClient,
+            CurrentDataHash = source.CurrentDataHash,
+            EnablePreDiscovery = source.EnablePreDiscovery,
+            UseParallelMethod = source.UseParallelMethod,
+            IsScheduledTransfer = source.IsScheduledTransfer,
+            SourceType = source.SourceType
+        };
+    }
+
    /// <summary>
    /// Verifica se un'associazione è stata creata dal Pre-Discovery
    /// controllando il campo AdditionalInfo
@@ -285,6 +508,22 @@ public interface IAssociationService
 {
    Task<KeyAssociation?> FindOrCreateAssociationAsync(PreDiscoveryRequest request);
    bool IsPreDiscoveryAssociation(KeyAssociation association);
+
+    /// <summary>
+    /// Versione bulk del find-or-create.
+    /// 1) Una sola query SQLite (WHERE KeyValue IN …) per recuperare le associazioni esistenti.
+    /// 2) Per le chiavi non trovate localmente, una manciata di SOQL "IN" su Salesforce
+    ///    (~200 chiavi per query, Composite API: ceil(K/25) HTTP call) invece di K chiamate singole.
+    /// 3) Le associazioni Pre-Discovery scoperte vengono salvate e restituite.
+    /// </summary>
+    /// <param name="sourceKeys">Lista (non vuota) dei valori chiave sorgente per tutti i record da analizzare.</param>
+    /// <param name="commonRequest">Parametri condivisi (entity, credential, restClient, mappings, ecc.).
+    ///   <see cref="PreDiscoveryRequest.SourceKey"/> e <see cref="PreDiscoveryRequest.CurrentDataHash"/>
+    ///   sono ignorati; vengono presi dal parametro <paramref name="sourceKeys"/>.</param>
+    /// <returns>Dizionario KeyValue → KeyAssociation (solo per chiavi trovate/create).</returns>
+    Task<Dictionary<string, KeyAssociation>> BatchFindOrCreateAssociationsAsync(
+        IEnumerable<string> sourceKeys,
+        PreDiscoveryRequest commonRequest);
 }

 /// <summary>