Alessio/Data-Coupler
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

[Feature] Salesforce: batch describe metadati, discovery parallela e fix scheduler External ID
Build and Push Docker Images / Build Linux Container (push) Successful in 6m56s
Details
Build and Push Docker Images / Build Windows Container (push) Has been cancelled
Details
Build and Push Docker Images / Create Multi-Platform Manifest (push) Has been cancelled
Details

Browse Source 
- Salesforce Composite Batch API per describe SObject: le describe sono ora
  raggruppate in chunk da 25 e inviate come singole POST a /composite/batch,
  riducendo le chiamate API da N a ceil(N/25); per 200 SObject: da 201 a 9 chiamate.

- Discovery entita' REST in parallelo: DiscoverEntitySummariesAsync e
  DiscoverEntitiesAsync avviate simultaneamente; la lista entita' diventa
  interattiva subito dopo le summaries, i dettagli completano in background
  con StateHasChanged() per aggiornare l'UI istantaneamente.

- Fix scheduler - preservazione ExternalIdRelationshipsJson e DefaultValuesJson:
  in DataCoupler.razor.cs entrambi i blocchi di update profilo esistente
  (riattivazione profilo inattivo e sovrascrittura profilo attivo) omettevano
  questi campi nella copia, causandone l'azzeramento silenzioso ad ogni
  re-salvataggio. Ora entrambi i percorsi propagano correttamente i campi JSON.

- Fix scheduler - esclusione campi sorgente External ID dal mapping normale:
  in ScheduledProfileExecutionService.TransformRecordForRest i campi sorgente
  usati nelle External ID Relationships venivano inclusi anche nel loop di
  field mapping standard, generando dati duplicati nell'entita' destinazione.
  Ora il comportamento e' allineato alla UI manuale (TransformRecordToRestEntity).

- Aggiornata documentazione: README.md, AGENTS.md, copilot-instructions.md
This commit was merged in pull request #11.
This commit is contained in:
Alessio Dal Santo
2026-02-20 14:59:13 +01:00
parent b1f83aa7ab
commit 335d587c89
9 changed files with 518 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DataConnection/REST/Implementations/SalesforceServiceClient.cs
+176 -56
View File
@@ -175,70 +175,43 @@ namespace DataConnection.REST.Implementations
var entities = new List<RestEntityInfo>();
try
{
// First, get list of all SObjects
// Step 1: get list of all SObjects (1 API call)
var sobjectsEndpoint = $"{_instanceUrl}/services/data/v60.0/sobjects/";
var response = await _httpClient.GetAsync(sobjectsEndpoint, cancellationToken);
response.EnsureSuccessStatusCode();
var sobjectsResponse = await response.Content.ReadFromJsonAsync<SalesforceSObjectsResponse>(cancellationToken: cancellationToken); if (sobjectsResponse?.SObjects != null)
var sobjectsResponse = await response.Content.ReadFromJsonAsync<SalesforceSObjectsResponse>(cancellationToken: cancellationToken);
if (sobjectsResponse?.SObjects != null)
{
// For demo purposes, limit to first 20 objects to avoid too many API calls
var limitedSObjects = sobjectsResponse.SObjects.ToList();
var sObjectNames = sobjectsResponse.SObjects
.Where(s => !string.IsNullOrEmpty(s.Name))
.Select(s => s.Name!)
.ToList();
// Process SObjects in parallel for better performance
var semaphore = new SemaphoreSlim(20, 20); // Limit concurrent requests to 5
var tasks = limitedSObjects.Where(sobject => !string.IsNullOrEmpty(sobject.Name))
.Select(async sobject =>
Console.WriteLine($"DiscoverEntities: {sObjectNames.Count} SObjects. Using Composite Batch API ({Math.Ceiling((double)sObjectNames.Count / 25)} request(s) instead of {sObjectNames.Count}).");
// Step 2: batch describe all SObjects via Composite Batch API (25 per request)
var describeResults = await BatchDescribeSObjectsAsync(sObjectNames, cancellationToken);
foreach (var sobject in sobjectsResponse.SObjects)
{
if (string.IsNullOrEmpty(sobject.Name)) continue;
if (!describeResults.TryGetValue(sobject.Name, out var describeResult) || describeResult?.Fields == null)
continue;
var entityInfo = new RestEntityInfo { Name = sobject.Name };
foreach (var field in describeResult.Fields)
{
await semaphore.WaitAsync(cancellationToken);
try
if (string.IsNullOrEmpty(field.Name)) continue;
entityInfo.Properties.Add(new RestPropertyInfo
{
// Get detailed field information for each SObject
var describeEndpoint = $"{_instanceUrl}/services/data/v60.0/sobjects/{sobject.Name}/describe/";
var describeResponse = await _httpClient.GetAsync(describeEndpoint, cancellationToken);
if (describeResponse.IsSuccessStatusCode)
{
var describeResult = await describeResponse.Content.ReadFromJsonAsync<SalesforceDescribeResponse>(cancellationToken: cancellationToken);
if (describeResult?.Fields != null)
{
var entityInfo = new RestEntityInfo
{
Name = sobject.Name
};
foreach (var field in describeResult.Fields)
{
if (string.IsNullOrEmpty(field.Name)) continue;
var propInfo = new RestPropertyInfo
{
Name = field.Name,
Type = field.Type ?? "string",
IsKey = field.Name.Equals("Id", StringComparison.OrdinalIgnoreCase)
};
entityInfo.Properties.Add(propInfo);
}
return entityInfo;
}
}
return null;
}
catch (Exception ex)
{
Console.WriteLine($"Error describing SObject {sobject.Name}: {ex.Message}");
return null;
}
finally
{
semaphore.Release();
}
});
var results = await Task.WhenAll(tasks);
entities.AddRange(results.Where(result => result != null)!);
Name = field.Name,
Type = field.Type ?? "string",
IsKey = field.Name.Equals("Id", StringComparison.OrdinalIgnoreCase)
});
}
entities.Add(entityInfo);
}
}
}
catch (HttpRequestException ex)
@@ -382,6 +355,116 @@ namespace DataConnection.REST.Implementations
return null;
}
/// <summary>
/// Describes multiple SObjects in batches using the Salesforce Composite Batch API.
/// Reduces API calls from N (one per object) to ceil(N/25) by grouping up to 25 describe
/// requests per Composite Batch call.
/// </summary>
private async Task<Dictionary<string, SalesforceDescribeResponse?>> BatchDescribeSObjectsAsync(
List<string> sObjectNames, CancellationToken cancellationToken)
{
const int maxBatchSize = 25;
var allResults = new Dictionary<string, SalesforceDescribeResponse?>(StringComparer.OrdinalIgnoreCase);
// Split into batches of 25 (Salesforce Composite Batch limit)
var batches = new List<(List<string> Names, int BatchNumber)>();
for (int i = 0; i < sObjectNames.Count; i += maxBatchSize)
{
var chunk = sObjectNames.Skip(i).Take(maxBatchSize).ToList();
batches.Add((chunk, (i / maxBatchSize) + 1));
}
Console.WriteLine($"BatchDescribeSObjects: {sObjectNames.Count} objects → {batches.Count} Composite Batch request(s)");
var batchEndpoint = $"{_instanceUrl}/services/data/v60.0/composite/batch";
// Execute all batches in parallel
var batchTasks = batches.Select(async b =>
{
Console.WriteLine($"BatchDescribeSObjects: sending batch {b.BatchNumber}/{batches.Count} ({b.Names.Count} objects)");
var batchRequest = new SalesforceBatchDescribeRequest
{
BatchRequests = b.Names.Select(name => new SalesforceBatchDescribeSubRequest
{
Method = "GET",
Url = $"/services/data/v60.0/sobjects/{name}/describe/"
}).ToList()
};
var jsonContent = new StringContent(
JsonSerializer.Serialize(batchRequest, SalesforceJsonOptions),
System.Text.Encoding.UTF8,
"application/json"
);
var batchResults = new Dictionary<string, SalesforceDescribeResponse?>(StringComparer.OrdinalIgnoreCase);
try
{
var response = await _httpClient.PostAsync(batchEndpoint, jsonContent, cancellationToken);
if (!response.IsSuccessStatusCode)
{
var err = await response.Content.ReadAsStringAsync(cancellationToken);
Console.WriteLine($"BatchDescribeSObjects batch {b.BatchNumber} failed: {response.StatusCode} - {err}");
foreach (var name in b.Names) batchResults[name] = null;
return batchResults;
}
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
var batchResponse = JsonSerializer.Deserialize<SalesforceBatchDescribeResponse>(responseContent, SalesforceJsonOptions);
if (batchResponse?.Results != null)
{
for (int i = 0; i < b.Names.Count; i++)
{
var objectName = b.Names[i];
if (i >= batchResponse.Results.Count)
{
batchResults[objectName] = null;
continue;
}
var subResponse = batchResponse.Results[i];
if (subResponse.StatusCode >= 200 && subResponse.StatusCode < 300 && subResponse.Result.HasValue)
{
try
{
batchResults[objectName] = JsonSerializer.Deserialize<SalesforceDescribeResponse>(
subResponse.Result.Value.GetRawText(), SalesforceJsonOptions);
}
catch (JsonException ex)
{
Console.WriteLine($"BatchDescribeSObjects: failed to parse describe for {objectName}: {ex.Message}");
batchResults[objectName] = null;
}
}
else
{
Console.WriteLine($"BatchDescribeSObjects: describe for {objectName} returned status {subResponse.StatusCode}");
batchResults[objectName] = null;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"BatchDescribeSObjects: exception in batch {b.BatchNumber}: {ex.Message}");
foreach (var name in b.Names) batchResults[name] = null;
}
return batchResults;
});
var allBatchResults = await Task.WhenAll(batchTasks);
foreach (var batchResult in allBatchResults)
foreach (var kvp in batchResult)
allResults[kvp.Key] = kvp.Value;
var successCount = allResults.Values.Count(v => v != null);
Console.WriteLine($"BatchDescribeSObjects completed: {successCount}/{sObjectNames.Count} objects described successfully.");
return allResults;
}
/// <summary>
/// Creates a new SObject in Salesforce.
/// </summary>
@@ -1631,6 +1714,43 @@ namespace DataConnection.REST.Implementations
public string? NextRecordsUrl { get; set; }
}
// ===== Composite Batch API models (for parallel describe calls) =====
private class SalesforceBatchDescribeRequest
{
[JsonPropertyName("batchRequests")]
public List<SalesforceBatchDescribeSubRequest> BatchRequests { get; set; } = new();
}
private class SalesforceBatchDescribeSubRequest
{
[JsonPropertyName("method")]
public string Method { get; set; } = string.Empty;
[JsonPropertyName("url")]
public string Url { get; set; } = string.Empty;
}
private class SalesforceBatchDescribeResponse
{
[JsonPropertyName("hasErrors")]
public bool HasErrors { get; set; }
[JsonPropertyName("results")]
public List<SalesforceBatchDescribeSubResponse> Results { get; set; } = new();
}
private class SalesforceBatchDescribeSubResponse
{
[JsonPropertyName("statusCode")]
public int StatusCode { get; set; }
[JsonPropertyName("result")]
public JsonElement? Result { get; set; }
}
// ===== Composite API models (for create/update/query operations) =====
private class SalesforceCompositeRequest
{
[JsonPropertyName("compositeRequest")]