Fix/query pagination wrapper

Generator/ClientExtensions.cs

@@ -1,11 +1,66 @@
 using Microsoft.PowerPlatform.Dataverse.Client;
+using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Messages;
 using Microsoft.Xrm.Sdk.Metadata;
+using Microsoft.Xrm.Sdk.Query;
 
 namespace Generator;
 
 public static class ClientExtensions
 {
+    /// <summary>
+    /// Retrieves all pages of records matching the query, automatically following Dataverse pagination.
+    /// Dataverse returns at most 5000 records per page by default; this wrapper collects all pages.
+    /// </summary>
+    public static async Task<List<Entity>> RetrieveAllAsync(this ServiceClient client, QueryExpression query, CancellationToken cancellationToken = default)
+    {
+        var results = new List<Entity>();
+        query.PageInfo = new PagingInfo
+        {
+            Count = 5000,
+            PageNumber = 1,
+            PagingCookie = null,
+            ReturnTotalRecordCount = false
+        };
+
+        EntityCollection response;
+        do
+        {
+            response = await client.RetrieveMultipleAsync(query, cancellationToken);
+            results.AddRange(response.Entities);
+            query.PageInfo.PageNumber++;
+            query.PageInfo.PagingCookie = response.PagingCookie;
+        } while (response.MoreRecords);
+
+        return results;
+    }
+
+    /// <summary>
+    /// Synchronous variant of RetrieveAllAsync for use in non-async contexts.
+    /// </summary>
+    public static List<Entity> RetrieveAll(this ServiceClient client, QueryExpression query)
+    {
+        var results = new List<Entity>();
+        query.PageInfo = new PagingInfo
+        {
+            Count = 5000,
+            PageNumber = 1,
+            PagingCookie = null,
+            ReturnTotalRecordCount = false
+        };
+
+        EntityCollection response;
+        do
+        {
+            response = client.RetrieveMultiple(query);
+            results.AddRange(response.Entities);
+            query.PageInfo.PageNumber++;
+            query.PageInfo.PagingCookie = response.PagingCookie;
+        } while (response.MoreRecords);
+
+        return results;
+    }
+
     public static async Task<EntityMetadata> RetrieveEntityAsync(this ServiceClient client, Guid objectId, CancellationToken token)
     {
         var resp = await client.ExecuteAsync(new RetrieveEntityRequest()

Generator/DataverseService.cs

@@ -82,8 +82,8 @@ public DataverseService(
             IEnumerable<ComponentInfo> solutionComponents;
             try
             {
-                logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponents()");
-                solutionComponents = solutionComponentService.GetAllSolutionComponents(solutionIds);
+                logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Calling solutionComponentService.GetAllSolutionComponentsAsync()");
+                solutionComponents = await solutionComponentService.GetAllSolutionComponentsAsync(solutionIds);
                 logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {solutionComponents.Count()} solution components");
             }
             catch (Exception ex)
@@ -219,7 +219,7 @@ public DataverseService(
 
                 // Get workflow dependencies for attributes (returns attribute ObjectId -> list of workflow ObjectIds)
                 var explicitComponentsList = solutionComponents.ToList();
-                var workflowDependencyMap = solutionComponentService.GetWorkflowDependenciesForAttributes(
+                var workflowDependencyMap = await solutionComponentService.GetWorkflowDependenciesForAttributesAsync(
                     explicitComponentsList.Where(c => c.ComponentType == 2).Select(c => new SolutionComponentInfo(
                         c.ObjectId,
                         c.SolutionComponentId ?? Guid.Empty,

Generator/Queries/PluginQueries.cs

@@ -71,9 +71,9 @@ public static async Task<IEnumerable<SDKStep>> GetSDKMessageProcessingStepsAsync
 
 
         //if (solutionIds is not null) query.Criteria.Conditions.Add(new ConditionExpression("solutionid", ConditionOperator.In, solutionIds));
-        var result = await service.RetrieveMultipleAsync(query);
+        var result = await service.RetrieveAllAsync(query);
 
-        var steps = result.Entities.Select(e =>
+        var steps = result.Select(e =>
         {
             var sdkMessageId = e.GetAttributeValue<AliasedValue>("step.sdkmessageid")?.Value as EntityReference;
             var sdkStepName = e.GetAttributeValue<AliasedValue>("step.name")?.Value as string;

Generator/Queries/PowerAutomateQueries.cs

@@ -42,14 +42,14 @@
             }
         };
 
-        var result = await service.RetrieveMultipleAsync(query);
+        var result = await service.RetrieveAllAsync(query);
 
-        var flows = result.Entities.Select(e =>
+        var flows = result.Select(e =>
         {
             return new PowerAutomateFlow(
                 e.GetAttributeValue<AliasedValue>("flow.workflowid").Value as string,
                 e.GetAttributeValue<AliasedValue>("flow.name").Value as string,
                 e.GetAttributeValue<AliasedValue>("flow.clientdata").Value as string
             );
         });
 

Generator/Queries/WebResourceQueries.cs

@@ -43,7 +43,7 @@
             }
         };
 
-        var results = (await service.RetrieveMultipleAsync(query)).Entities;
+        var results = await service.RetrieveAllAsync(query);
 
         var webResources = results.Select(e =>
         {
@@ -73,7 +73,7 @@
 
             return new WebResource(
                 webresourceId,
                 webresourceName,
                 content,
                 (OptionSetValue)e.GetAttributeValue<AliasedValue>("webresource.webresourcetype").Value,
                 webresourceDesc

Generator/Services/EntityIconService.cs

@@ -43,8 +43,8 @@ public async Task<Dictionary<string, string>> GetEntityIconMap(IEnumerable<Entit
                     }
                 };
 
-                var webresources = await client.RetrieveMultipleAsync(query);
-                iconNameToSvg = webresources.Entities.ToDictionary(x => x.GetAttributeValue<string>("name"), x => x.GetAttributeValue<string>("content"));
+                var webresources = await client.RetrieveAllAsync(query);
+                iconNameToSvg = webresources.ToDictionary(x => x.GetAttributeValue<string>("name"), x => x.GetAttributeValue<string>("content"));
             }
 
             var logicalNameToSvg =

Generator/Services/SecurityRoleService.cs

@@ -64,9 +64,9 @@ public async Task<Dictionary<string, List<SecurityRole>>> GetSecurityRoles(
                 }
             };
 
-            var roles = await client.RetrieveMultipleAsync(query);
+            var roles = await client.RetrieveAllAsync(query);
 
-            var rolePrivileges = roles.Entities.Select(e =>
+            var rolePrivileges = roles.Select(e =>
             {
                 var name = e.GetAttributeValue<string>("name");
                 var depth = (PrivilegeDepth)e.GetAttributeValue<AliasedValue>("rolepriv.privilegedepthmask").Value;

Generator/Services/SolutionComponentExtractor.cs

@@ -170,8 +170,8 @@ private async Task<List<RawComponentInfo>> QuerySolutionComponentsAsync(List<Gui
 
         try
         {
-            var response = await _client.RetrieveMultipleAsync(query);
-            foreach (var entity in response.Entities)
+            var entities = await _client.RetrieveAllAsync(query);
+            foreach (var entity in entities)
             {
                 var componentType = entity.GetAttributeValue<OptionSetValue>("componenttype")?.Value ?? 0;
                 var objectId = entity.GetAttributeValue<Guid>("objectid");
@@ -315,8 +315,8 @@ private async Task<List<RawComponentInfo>> QuerySolutionComponentsAsync(List<Gui
                 }
             };
 
-            var response = await _client.RetrieveMultipleAsync(query);
-            foreach (var entity in response.Entities)
+            var entities = await _client.RetrieveAllAsync(query);
+            foreach (var entity in entities)
             {
                 var id = entity.GetAttributeValue<Guid>(primaryKey);
                 var name = entity.GetAttributeValue<string>(nameColumn) ?? id.ToString();

Generator/Services/SolutionComponentService.cs

@@ -65,7 +65,7 @@ public SolutionComponentService(ServiceClient client, IConfiguration configurati
         _logger = logger;
     }
 
-    public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionIds)
+    public async Task<IEnumerable<ComponentInfo>> GetAllSolutionComponentsAsync(List<Guid> solutionIds)
     {
         _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Getting all solution components for solutions: {string.Join(", ", solutionIds)}");
         var allComponents = new HashSet<ComponentInfo>();
@@ -74,7 +74,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
         IEnumerable<SolutionComponentInfo> explicitComponents;
         try
         {
-            explicitComponents = GetExplicitComponents(solutionIds);
+            explicitComponents = await GetExplicitComponentsAsync(solutionIds);
             _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Found {explicitComponents.Count()} explicit components");
         }
         catch (Exception ex)
@@ -118,7 +118,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
                 _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieving component information for {requiredNodeIds.Count} dependency nodes");
 
                 // Retrieve component node information
-                var componentNodes = GetComponentNodes(requiredNodeIds);
+                var componentNodes = await GetComponentNodesAsync(requiredNodeIds);
                 _logger.LogInformation($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] Retrieved {componentNodes.Count} component nodes");
 
                 // Add required components as implicit
@@ -152,7 +152,7 @@ public IEnumerable<ComponentInfo> GetAllSolutionComponents(List<Guid> solutionId
         return allComponents;
     }
 
-    private IEnumerable<SolutionComponentInfo> GetExplicitComponents(List<Guid> solutionIds)
+    private async Task<IEnumerable<SolutionComponentInfo>> GetExplicitComponentsAsync(List<Guid> solutionIds)
     {
         var query = new QueryExpression("solutioncomponent")
         {
@@ -167,16 +167,16 @@ private IEnumerable<SolutionComponentInfo> GetExplicitComponents(List<Guid> solu
             }
         };
 
-        return _client.RetrieveMultiple(query).Entities
-                .Select(e => new SolutionComponentInfo(
-                    e.GetAttributeValue<Guid>("objectid"),
-                    e.GetAttributeValue<Guid>("solutioncomponentid"),
-                    e.GetAttributeValue<OptionSetValue>("componenttype").Value,
-                    e.Contains("rootcomponentbehavior") ? e.GetAttributeValue<OptionSetValue>("rootcomponentbehavior").Value : -1,
-                    e.GetAttributeValue<EntityReference>("solutionid")));
+        var entities = await _client.RetrieveAllAsync(query);
+        return entities.Select(e => new SolutionComponentInfo(
+            e.GetAttributeValue<Guid>("objectid"),
+            e.GetAttributeValue<Guid>("solutioncomponentid"),
+            e.GetAttributeValue<OptionSetValue>("componenttype").Value,
+            e.Contains("rootcomponentbehavior") ? e.GetAttributeValue<OptionSetValue>("rootcomponentbehavior").Value : -1,
+            e.GetAttributeValue<EntityReference>("solutionid")));
     }
 
-    private List<ComponentNodeInfo> GetComponentNodes(List<Guid> nodeIds)
+    private async Task<List<ComponentNodeInfo>> GetComponentNodesAsync(List<Guid> nodeIds)
     {
         if (!nodeIds.Any())
         {
@@ -198,8 +198,8 @@ private List<ComponentNodeInfo> GetComponentNodes(List<Guid> nodeIds)
 
         try
         {
-            var response = _client.RetrieveMultiple(query);
-            foreach (var entity in response.Entities)
+            var entities = await _client.RetrieveAllAsync(query);
+            foreach (var entity in entities)
             {
                 results.Add(new ComponentNodeInfo(
                     entity.GetAttributeValue<Guid>("dependencynodeid"),
@@ -374,7 +374,7 @@ private HashSet<DependencyInfo> GetRequiredComponents(IEnumerable<SolutionCompon
     /// </summary>
     /// <param name="attributeComponents">List of attribute components to check for dependencies</param>
     /// <returns>Dictionary mapping attribute ObjectId to list of workflow ObjectIds that depend on it</returns>
-    public Dictionary<Guid, List<Guid>> GetWorkflowDependenciesForAttributes(IEnumerable<SolutionComponentInfo> attributeComponents)
+    public async Task<Dictionary<Guid, List<Guid>>> GetWorkflowDependenciesForAttributesAsync(IEnumerable<SolutionComponentInfo> attributeComponents)
     {
         var workflowDependencies = new Dictionary<Guid, List<Guid>>();
 
@@ -406,7 +406,7 @@ public Dictionary<Guid, List<Guid>> GetWorkflowDependenciesForAttributes(IEnumer
             return workflowDependencies;
 
         // Retrieve component node information for all nodes
-        var allNodes = GetComponentNodes(allNodeIds);
+        var allNodes = await GetComponentNodesAsync(allNodeIds);
 
         // Filter to only workflow components (type 29)
         var workflowNodes = allNodes.Where(n => n.ComponentType == 29).ToList();

Generator/Services/SolutionService.cs

@@ -34,7 +34,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
             }
             var solutionNames = solutionNameArg.Split(",").Select(x => x.Trim().ToLower()).ToList();
 
-            var resp = await client.RetrieveMultipleAsync(new QueryExpression("solution")
+            var entities = await client.RetrieveAllAsync(new QueryExpression("solution")
             {
                 ColumnSet = new ColumnSet("publisherid", "friendlyname", "uniquename", "solutionid"),
                 Criteria = new FilterExpression(LogicalOperator.And)
@@ -46,7 +46,7 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
                 }
             });
 
-            return (resp.Entities.Select(e => e.GetAttributeValue<Guid>("solutionid")).ToList(), resp.Entities.ToList());
+            return (entities.Select(e => e.GetAttributeValue<Guid>("solutionid")).ToList(), entities);
         }
 
         /// <summary>
@@ -73,8 +73,8 @@ public SolutionService(ServiceClient client, IConfiguration configuration, ILogg
                 }
             };
 
-            var publishers = await client.RetrieveMultipleAsync(publisherQuery);
-            return publishers.Entities.ToDictionary(
+            var publishers = await client.RetrieveAllAsync(publisherQuery);
+            return publishers.ToDictionary(
                 p => p.GetAttributeValue<Guid>("publisherid"),
                 p => (
                     Name: p.GetAttributeValue<string>("friendlyname") ?? "Unknown Publisher",

Generator/Services/WorkflowService.cs

@@ -38,9 +38,9 @@ public async Task<Dictionary<Guid, WorkflowInfo>> GetWorkflows(List<Guid> workfl
                 }
             };
 
-            var results = await client.RetrieveMultipleAsync(query);
+            var results = await client.RetrieveAllAsync(query);
 
-            return results.Entities.ToDictionary(
+            return results.ToDictionary(
                 e => e.Id,
                 e => new WorkflowInfo(
                     e.Id,

Generator/WebsiteBuilder.cs

@@ -2,7 +2,6 @@
 using Generator.DTO.Warnings;
 using Microsoft.Extensions.Configuration;
 using Newtonsoft.Json;
-using System.Collections.Generic;
 using System.Text;
 
 namespace Generator;
@@ -64,7 +63,7 @@ internal void AddData()
             sb.AppendLine("  },");
         }
 
-        sb.AppendLine("]");
+        sb.AppendLine("] as const;");
 
         // WARNINGS
         sb.AppendLine("");
@@ -73,7 +72,7 @@ internal void AddData()
         {
             sb.AppendLine($"  {JsonConvert.SerializeObject(warning)},");
         }
-        sb.AppendLine("]");
+        sb.AppendLine("] as const;");
 
         // SOLUTION COMPONENTS (for insights)
         sb.AppendLine("");