diff --git a/Generator/ClientExtensions.cs b/Generator/ClientExtensions.cs index d19d4eb..8a2dfca 100644 --- a/Generator/ClientExtensions.cs +++ b/Generator/ClientExtensions.cs @@ -1,11 +1,66 @@ using Microsoft.PowerPlatform.Dataverse.Client; +using Microsoft.Xrm.Sdk; using Microsoft.Xrm.Sdk.Messages; using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; namespace Generator; public static class ClientExtensions { + /// + /// Retrieves all pages of records matching the query, automatically following Dataverse pagination. + /// Dataverse returns at most 5000 records per page by default; this wrapper collects all pages. + /// + public static async Task> RetrieveAllAsync(this ServiceClient client, QueryExpression query, CancellationToken cancellationToken = default) + { + var results = new List(); + query.PageInfo = new PagingInfo + { + Count = 5000, + PageNumber = 1, + PagingCookie = null, + ReturnTotalRecordCount = false + }; + + EntityCollection response; + do + { + response = await client.RetrieveMultipleAsync(query, cancellationToken); + results.AddRange(response.Entities); + query.PageInfo.PageNumber++; + query.PageInfo.PagingCookie = response.PagingCookie; + } while (response.MoreRecords); + + return results; + } + + /// + /// Synchronous variant of RetrieveAllAsync for use in non-async contexts. + /// + public static List RetrieveAll(this ServiceClient client, QueryExpression query) + { + var results = new List(); + query.PageInfo = new PagingInfo + { + Count = 5000, + PageNumber = 1, + PagingCookie = null, + ReturnTotalRecordCount = false + }; + + EntityCollection response; + do + { + response = client.RetrieveMultiple(query); + results.AddRange(response.Entities); + query.PageInfo.PageNumber++; + query.PageInfo.PagingCookie = response.PagingCookie; + } while (response.MoreRecords); + + return results; + } + public static async Task RetrieveEntityAsync(this ServiceClient client, Guid objectId, CancellationToken token) { var resp = await client.ExecuteAsync(new RetrieveEntityRequest() diff --git a/Generator/DataverseService.cs b/Generator/DataverseService.cs index 6c70df1..555b294 100644 --- a/Generator/DataverseService.cs +++ b/Generator/DataverseService.cs @@ -82,8 +82,8 @@ public DataverseService( IEnumerable solutionComponents; try { - logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponents()"); - solutionComponents = solutionComponentService.GetAllSolutionComponents(solutionIds); + logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponentsAsync()"); + solutionComponents = await solutionComponentService.GetAllSolutionComponentsAsync(solutionIds); logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {solutionComponents.Count()} solution components"); } catch (Exception ex) @@ -219,7 +219,7 @@ public DataverseService( // Get workflow dependencies for attributes (returns attribute ObjectId -> list of workflow ObjectIds) var explicitComponentsList = solutionComponents.ToList(); - var workflowDependencyMap = solutionComponentService.GetWorkflowDependenciesForAttributes( + var workflowDependencyMap = await solutionComponentService.GetWorkflowDependenciesForAttributesAsync( explicitComponentsList.Where(c => c.ComponentType == 2).Select(c => new SolutionComponentInfo( c.ObjectId, c.SolutionComponentId ?? Guid.Empty, diff --git a/Generator/Queries/PluginQueries.cs b/Generator/Queries/PluginQueries.cs index 1955a99..a0221b4 100644 --- a/Generator/Queries/PluginQueries.cs +++ b/Generator/Queries/PluginQueries.cs @@ -71,9 +71,9 @@ public static async Task> GetSDKMessageProcessingStepsAsync //if (solutionIds is not null) query.Criteria.Conditions.Add(new ConditionExpression("solutionid", ConditionOperator.In, solutionIds)); - var result = await service.RetrieveMultipleAsync(query); + var result = await service.RetrieveAllAsync(query); - var steps = result.Entities.Select(e => + var steps = result.Select(e => { var sdkMessageId = e.GetAttributeValue("step.sdkmessageid")?.Value as EntityReference; var sdkStepName = e.GetAttributeValue("step.name")?.Value as string; diff --git a/Generator/Queries/PowerAutomateQueries.cs b/Generator/Queries/PowerAutomateQueries.cs index d20b302..cc6dc80 100644 --- a/Generator/Queries/PowerAutomateQueries.cs +++ b/Generator/Queries/PowerAutomateQueries.cs @@ -42,9 +42,9 @@ public static async Task> GetPowerAutomateFlowsAs } }; - var result = await service.RetrieveMultipleAsync(query); + var result = await service.RetrieveAllAsync(query); - var flows = result.Entities.Select(e => + var flows = result.Select(e => { return new PowerAutomateFlow( e.GetAttributeValue("flow.workflowid").Value as string, diff --git a/Generator/Queries/WebResourceQueries.cs b/Generator/Queries/WebResourceQueries.cs index f1be285..050316e 100644 --- a/Generator/Queries/WebResourceQueries.cs +++ b/Generator/Queries/WebResourceQueries.cs @@ -43,7 +43,7 @@ public static async Task> GetWebResourcesAsync(this Ser } }; - var results = (await service.RetrieveMultipleAsync(query)).Entities; + var results = await service.RetrieveAllAsync(query); var webResources = results.Select(e => { diff --git a/Generator/Services/EntityIconService.cs b/Generator/Services/EntityIconService.cs index 853c59a..5003860 100644 --- a/Generator/Services/EntityIconService.cs +++ b/Generator/Services/EntityIconService.cs @@ -43,8 +43,8 @@ public async Task> GetEntityIconMap(IEnumerable x.GetAttributeValue("name"), x => x.GetAttributeValue("content")); + var webresources = await client.RetrieveAllAsync(query); + iconNameToSvg = webresources.ToDictionary(x => x.GetAttributeValue("name"), x => x.GetAttributeValue("content")); } var logicalNameToSvg = diff --git a/Generator/Services/SecurityRoleService.cs b/Generator/Services/SecurityRoleService.cs index 834aadd..1f0824e 100644 --- a/Generator/Services/SecurityRoleService.cs +++ b/Generator/Services/SecurityRoleService.cs @@ -64,9 +64,9 @@ public async Task>> GetSecurityRoles( } }; - var roles = await client.RetrieveMultipleAsync(query); + var roles = await client.RetrieveAllAsync(query); - var rolePrivileges = roles.Entities.Select(e => + var rolePrivileges = roles.Select(e => { var name = e.GetAttributeValue("name"); var depth = (PrivilegeDepth)e.GetAttributeValue("rolepriv.privilegedepthmask").Value; diff --git a/Generator/Services/SolutionComponentExtractor.cs b/Generator/Services/SolutionComponentExtractor.cs index 5d4fc66..6fe0ceb 100644 --- a/Generator/Services/SolutionComponentExtractor.cs +++ b/Generator/Services/SolutionComponentExtractor.cs @@ -170,8 +170,8 @@ private async Task> QuerySolutionComponentsAsync(List("componenttype")?.Value ?? 0; var objectId = entity.GetAttributeValue("objectid"); @@ -315,8 +315,8 @@ private async Task> QuerySolutionComponentsAsync(List(primaryKey); var name = entity.GetAttributeValue(nameColumn) ?? id.ToString(); diff --git a/Generator/Services/SolutionComponentService.cs b/Generator/Services/SolutionComponentService.cs index a7e7ed9..895c52a 100644 --- a/Generator/Services/SolutionComponentService.cs +++ b/Generator/Services/SolutionComponentService.cs @@ -65,7 +65,7 @@ public SolutionComponentService(ServiceClient client, IConfiguration configurati _logger = logger; } - public IEnumerable GetAllSolutionComponents(List solutionIds) + public async Task> GetAllSolutionComponentsAsync(List solutionIds) { _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Getting all solution components for solutions: {string.Join(", ", solutionIds)}"); var allComponents = new HashSet(); @@ -74,7 +74,7 @@ public IEnumerable GetAllSolutionComponents(List solutionId IEnumerable explicitComponents; try { - explicitComponents = GetExplicitComponents(solutionIds); + explicitComponents = await GetExplicitComponentsAsync(solutionIds); _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {explicitComponents.Count()} explicit components"); } catch (Exception ex) @@ -118,7 +118,7 @@ public IEnumerable GetAllSolutionComponents(List solutionId _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving component information for {requiredNodeIds.Count} dependency nodes"); // Retrieve component node information - var componentNodes = GetComponentNodes(requiredNodeIds); + var componentNodes = await GetComponentNodesAsync(requiredNodeIds); _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {componentNodes.Count} component nodes"); // Add required components as implicit @@ -152,7 +152,7 @@ public IEnumerable GetAllSolutionComponents(List solutionId return allComponents; } - private IEnumerable GetExplicitComponents(List solutionIds) + private async Task> GetExplicitComponentsAsync(List solutionIds) { var query = new QueryExpression("solutioncomponent") { @@ -167,16 +167,16 @@ private IEnumerable GetExplicitComponents(List solu } }; - return _client.RetrieveMultiple(query).Entities - .Select(e => new SolutionComponentInfo( - e.GetAttributeValue("objectid"), - e.GetAttributeValue("solutioncomponentid"), - e.GetAttributeValue("componenttype").Value, - e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, - e.GetAttributeValue("solutionid"))); + var entities = await _client.RetrieveAllAsync(query); + return entities.Select(e => new SolutionComponentInfo( + e.GetAttributeValue("objectid"), + e.GetAttributeValue("solutioncomponentid"), + e.GetAttributeValue("componenttype").Value, + e.Contains("rootcomponentbehavior") ? e.GetAttributeValue("rootcomponentbehavior").Value : -1, + e.GetAttributeValue("solutionid"))); } - private List GetComponentNodes(List nodeIds) + private async Task> GetComponentNodesAsync(List nodeIds) { if (!nodeIds.Any()) { @@ -198,8 +198,8 @@ private List GetComponentNodes(List nodeIds) try { - var response = _client.RetrieveMultiple(query); - foreach (var entity in response.Entities) + var entities = await _client.RetrieveAllAsync(query); + foreach (var entity in entities) { results.Add(new ComponentNodeInfo( entity.GetAttributeValue("dependencynodeid"), @@ -374,7 +374,7 @@ private HashSet GetRequiredComponents(IEnumerable /// List of attribute components to check for dependencies /// Dictionary mapping attribute ObjectId to list of workflow ObjectIds that depend on it - public Dictionary> GetWorkflowDependenciesForAttributes(IEnumerable attributeComponents) + public async Task>> GetWorkflowDependenciesForAttributesAsync(IEnumerable attributeComponents) { var workflowDependencies = new Dictionary>(); @@ -406,7 +406,7 @@ public Dictionary> GetWorkflowDependenciesForAttributes(IEnumer return workflowDependencies; // Retrieve component node information for all nodes - var allNodes = GetComponentNodes(allNodeIds); + var allNodes = await GetComponentNodesAsync(allNodeIds); // Filter to only workflow components (type 29) var workflowNodes = allNodes.Where(n => n.ComponentType == 29).ToList(); diff --git a/Generator/Services/SolutionService.cs b/Generator/Services/SolutionService.cs index 80b4f62..475acec 100644 --- a/Generator/Services/SolutionService.cs +++ b/Generator/Services/SolutionService.cs @@ -34,7 +34,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg } var solutionNames = solutionNameArg.Split(",").Select(x => x.Trim().ToLower()).ToList(); - var resp = await client.RetrieveMultipleAsync(new QueryExpression("solution") + var entities = await client.RetrieveAllAsync(new QueryExpression("solution") { ColumnSet = new ColumnSet("publisherid", "friendlyname", "uniquename", "solutionid"), Criteria = new FilterExpression(LogicalOperator.And) @@ -46,7 +46,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg } }); - return (resp.Entities.Select(e => e.GetAttributeValue("solutionid")).ToList(), resp.Entities.ToList()); + return (entities.Select(e => e.GetAttributeValue("solutionid")).ToList(), entities); } /// @@ -73,8 +73,8 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg } }; - var publishers = await client.RetrieveMultipleAsync(publisherQuery); - return publishers.Entities.ToDictionary( + var publishers = await client.RetrieveAllAsync(publisherQuery); + return publishers.ToDictionary( p => p.GetAttributeValue("publisherid"), p => ( Name: p.GetAttributeValue("friendlyname") ?? "Unknown Publisher", diff --git a/Generator/Services/WorkflowService.cs b/Generator/Services/WorkflowService.cs index c3cac7f..da1ef5e 100644 --- a/Generator/Services/WorkflowService.cs +++ b/Generator/Services/WorkflowService.cs @@ -38,9 +38,9 @@ public async Task> GetWorkflows(List workfl } }; - var results = await client.RetrieveMultipleAsync(query); + var results = await client.RetrieveAllAsync(query); - return results.Entities.ToDictionary( + return results.ToDictionary( e => e.Id, e => new WorkflowInfo( e.Id, diff --git a/Generator/WebsiteBuilder.cs b/Generator/WebsiteBuilder.cs index f443ec7..4cbe5b4 100644 --- a/Generator/WebsiteBuilder.cs +++ b/Generator/WebsiteBuilder.cs @@ -2,7 +2,6 @@ using Generator.DTO.Warnings; using Microsoft.Extensions.Configuration; using Newtonsoft.Json; -using System.Collections.Generic; using System.Text; namespace Generator; @@ -64,7 +63,7 @@ internal void AddData() sb.AppendLine(" },"); } - sb.AppendLine("]"); + sb.AppendLine("] as const;"); // WARNINGS sb.AppendLine(""); @@ -73,7 +72,7 @@ internal void AddData() { sb.AppendLine($" {JsonConvert.SerializeObject(warning)},"); } - sb.AppendLine("]"); + sb.AppendLine("] as const;"); // SOLUTION COMPONENTS (for insights) sb.AppendLine("");