Merge pull request #258 from atc0005/i245-fix-paging-collection-logic

Fix paging collection logic

Author: atc0005

internal/rsat/organizations.go

@@ -9,6 +9,7 @@ package rsat
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strconv"
@@ -41,7 +42,12 @@ type OrganizationsResponse struct {
 	Total int `json:"total"`
 
 	// Page is the page number for the current query response results.
-	Page int `json:"page"`
+	//
+	// NOTE: In practice, this value has been found to be  returned as an
+	// integer in the first response and as a string value for each additional
+	// page of results. The json.Number type accepts either format when
+	// decoding the response.
+	Page json.Number `json:"page"`
 
 	// PerPage is the pagination limit applied to API query results. If not
 	// specified by the client this is the default value set by the API.
@@ -131,6 +137,8 @@ func GetOrganizations(ctx context.Context, client *APIClient) ([]Organization, e
 			logger.Error().Err(closeErr).Msg("error closing response body")
 		}
 
+		allOrgs = append(allOrgs, orgsQueryResp.Organizations...)
+
 		numNewOrgs := len(orgsQueryResp.Organizations)
 		numCollectedOrgs := len(allOrgs)
 		numOrgsRemaining := orgsQueryResp.Subtotal - numCollectedOrgs

internal/rsat/rsat.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 
 	"github.com/rs/zerolog"
 )
@@ -157,7 +158,17 @@ func decode(dst interface{}, reader io.Reader, logger zerolog.Logger, sourceName
 		sourceName,
 		limit,
 	)
-	dec := json.NewDecoder(io.LimitReader(reader, limit))
+
+	limitReader := io.LimitReader(reader, limit)
+
+	// If debug or greater logging is enabled write the JSON payload in the
+	// response as-is to stderr for review.
+	if zerolog.GlobalLevel() == zerolog.DebugLevel ||
+		zerolog.GlobalLevel() == zerolog.TraceLevel {
+		limitReader = io.TeeReader(io.LimitReader(reader, limit), os.Stderr)
+	}
+
+	dec := json.NewDecoder(limitReader)
 
 	// This project does not use all fields from Red Hat Satellite API
 	// responses so we do not attempt to assert that we've accounted for all

internal/rsat/syncplans.go

@@ -9,6 +9,7 @@ package rsat
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
 	"strconv"
@@ -49,7 +50,12 @@ type SyncPlansResponse struct {
 	Total int `json:"total"`
 
 	// Page is the page number for the current query response results.
-	Page int `json:"page"`
+	//
+	// NOTE: In practice, this value has been found to be  returned as an
+	// integer in the first response and as a string value for each additional
+	// page of results. The json.Number type accepts either format when
+	// decoding the response.
+	Page json.Number `json:"page"`
 
 	// PerPage is the pagination limit applied to API query results. If not
 	// specified by the client this is the default value set by the API.
@@ -467,12 +473,12 @@ func getOrgSyncPlans(ctx context.Context, client *APIClient, org Organization) (
 			syncPlansQueryResp.SyncPlans[i].OrganizationTitle = org.Title
 		}
 
+		allSyncPlans = append(allSyncPlans, syncPlansQueryResp.SyncPlans...)
+
 		numNewSyncPlans := len(syncPlansQueryResp.SyncPlans)
 		numCollectedSyncPlans := len(allSyncPlans)
 		numSyncPlansRemaining := syncPlansQueryResp.Subtotal - numCollectedSyncPlans
 
-		allSyncPlans = append(allSyncPlans, syncPlansQueryResp.SyncPlans...)
-
 		subLogger.Debug().
 			Str("api_endpoint", apiURL).
 			Int("sync_plans_collected", numCollectedSyncPlans).