fix(rabbitmq): add bounded paging for queue and exchange management lists

Closes #8879
This commit is contained in:
t8y2 committed 2026-09-27 09:38:54 +00:00
1 parent 33f212a452
commit 725b1ee252
22 files changed
+1445 -67

No files matched your search

+219
View File
@@ -11,6 +11,9 @@ import (
"io"
"net"
"net/http"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -21,10 +24,28 @@ const (
defaultManagementTLSPort = 15671
defaultAMQPTLSPort = 5671
managementPageSize = 100
managementMaxPageSize = 500
managementConnectTimeout = 10 * time.Second
managementRequestTimeout = 20 * time.Second
)
type managementPageRequest struct {
Page int
PageSize int
Search string
UseRegex bool
Sort string
SortReverse bool
}
type managementPageResult struct {
Items []any
Page int
PageSize int
TotalCount int
HasMore bool
}
type managementStatusError struct {
status int
method string
@@ -172,6 +193,204 @@ func managementGetAll(connection jsonObject, path string) ([]any, error) {
}
}
// managementGetPage requests exactly one RabbitMQ management API page. Queue
// and exchange endpoints in RabbitMQ 3.8.x return a pagination envelope when
// page/page_size are present. The plain-array fallback keeps the agent usable
// with endpoints or older brokers that ignore those parameters, while still
// presenting the same explicit page contract to callers.
func managementGetPage(connection jsonObject, path string, request managementPageRequest) (managementPageResult, error) {
if err := validateManagementPageRequest(request); err != nil {
return managementPageResult{}, err
}
query := url.Values{}
query.Set("page", strconv.Itoa(request.Page))
query.Set("page_size", strconv.Itoa(request.PageSize))
if request.Search != "" {
query.Set("name", request.Search)
query.Set("use_regex", strconv.FormatBool(request.UseRegex))
}
if request.Sort != "" {
query.Set("sort", request.Sort)
}
if request.SortReverse {
query.Set("sort_reverse", "true")
}
response, err := managementGet(connection, appendManagementQuery(path, query))
if err != nil {
return managementPageResult{}, err
}
switch typed := response.(type) {
case []any:
return paginateManagementArray(typed, request)
case map[string]any:
object := jsonObject(typed)
items, exists := typed["items"]
if !exists {
return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path)
}
array, ok := items.([]any)
if !ok {
return managementPageResult{}, fmt.Errorf("Unexpected management API items for list endpoint %s", path)
}
page := intOrDefault(object, "page", request.Page)
pageSize := intOrDefault(object, "page_size", request.PageSize)
totalCount := integerOrNull(object, "filtered_count")
if totalCount == nil {
totalCount = integerOrNull(object, "total_count")
}
if totalCount == nil {
inferred := (page-1)*pageSize + len(array)
totalCount = &inferred
}
pageCount := integerOrNull(object, "page_count")
hasMore := pageCount != nil && page < *pageCount
if pageCount == nil {
hasMore = page*pageSize < *totalCount
}
return managementPageResult{
Items: array,
Page: page,
PageSize: pageSize,
TotalCount: *totalCount,
HasMore: hasMore,
}, nil
default:
return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path)
}
}
func managementPageRequestFromParams(params jsonObject, allowedSorts ...string) (*managementPageRequest, error) {
page := integerOrNull(params, "page")
pageSize := integerOrNull(params, "page_size")
if page == nil && pageSize == nil {
return nil, nil
}
if page == nil || pageSize == nil {
return nil, errors.New("page and page_size must be provided together")
}
sortName := stringOrDefault(params, "sort", "name")
allowed := false
for _, candidate := range allowedSorts {
if sortName == candidate {
allowed = true
break
}
}
if !allowed {
return nil, fmt.Errorf("Unsupported list sort %q", sortName)
}
request := managementPageRequest{
Page: *page,
PageSize: *pageSize,
Search: strings.TrimSpace(stringOrEmpty(params, "search")),
Sort: sortName,
SortReverse: boolOrDefault(params, "sort_reverse", false),
}
if err := validateManagementPageRequest(request); err != nil {
return nil, err
}
return &request, nil
}
func validateManagementPageRequest(request managementPageRequest) error {
if request.Page < 1 {
return errors.New("page must be at least 1")
}
if request.PageSize < 1 || request.PageSize > managementMaxPageSize {
return fmt.Errorf("page_size must be between 1 and %d", managementMaxPageSize)
}
maxInt := int(^uint(0) >> 1)
if request.Page-1 > maxInt/request.PageSize {
return errors.New("page is too large")
}
return nil
}
func appendManagementQuery(path string, values url.Values) string {
separator := "?"
if strings.Contains(path, "?") {
separator = "&"
}
return path + separator + values.Encode()
}
func paginateManagementArray(items []any, request managementPageRequest) (managementPageResult, error) {
filtered := make([]any, 0, len(items))
var expression *regexp.Regexp
if request.Search != "" && request.UseRegex {
compiled, err := regexp.Compile("(?i)" + request.Search)
if err != nil {
return managementPageResult{}, fmt.Errorf("Invalid management list search: %w", err)
}
expression = compiled
}
search := strings.ToLower(request.Search)
for _, item := range items {
if request.Search != "" {
object, ok := item.(map[string]any)
if !ok {
continue
}
name := stringOrEmpty(jsonObject(object), "name")
if expression != nil {
if !expression.MatchString(name) {
continue
}
} else if !strings.Contains(strings.ToLower(name), search) {
continue
}
}
filtered = append(filtered, item)
}
sort.SliceStable(filtered, func(left, right int) bool {
leftObject, _ := filtered[left].(map[string]any)
rightObject, _ := filtered[right].(map[string]any)
leftJSON := jsonObject(leftObject)
rightJSON := jsonObject(rightObject)
comparison := 0
if request.Sort == "messages_ready" {
leftCount := longOrDefault(leftJSON, "messages_ready", 0)
rightCount := longOrDefault(rightJSON, "messages_ready", 0)
if leftCount < rightCount {
comparison = -1
} else if leftCount > rightCount {
comparison = 1
}
}
if comparison == 0 {
comparison = strings.Compare(stringOrEmpty(leftJSON, "name"), stringOrEmpty(rightJSON, "name"))
}
if request.SortReverse {
comparison = -comparison
}
return comparison < 0
})
totalCount := len(filtered)
start := (request.Page - 1) * request.PageSize
if start >= totalCount {
return managementPageResult{
Items: []any{},
Page: request.Page,
PageSize: request.PageSize,
TotalCount: totalCount,
HasMore: false,
}, nil
}
end := start + request.PageSize
if end > totalCount {
end = totalCount
}
return managementPageResult{
Items: filtered[start:end],
Page: request.Page,
PageSize: request.PageSize,
TotalCount: totalCount,
HasMore: end < totalCount,
}, nil
}
func managementBaseURLs(connection jsonObject) ([]string, error) {
if explicit := stringOrNull(connection, "management_url"); explicit != nil && strings.TrimSpace(*explicit) != "" {
return []string{normalizeManagementURL(*explicit)}, nil
+128
View File
@@ -135,6 +135,134 @@ func TestManagementGetAllPlainArray(t *testing.T) {
}
}
func TestManagementGetPageRequestsOnlyOneRabbitMqPage(t *testing.T) {
requestCount := 0
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requestCount++
query := request.URL.Query()
if query.Get("page") != "2" || query.Get("page_size") != "100" {
t.Fatalf("unexpected pagination query %q", request.URL.RawQuery)
}
if query.Get("name") != "orders & paid" || query.Get("use_regex") != "false" {
t.Fatalf("unexpected search query %q", request.URL.RawQuery)
}
if query.Get("sort") != "messages_ready" || query.Get("sort_reverse") != "true" {
t.Fatalf("unexpected sort query %q", request.URL.RawQuery)
}
writer.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(writer, `{"items":[{"name":"orders.paid"}],"page":2,"page_size":100,"page_count":3,"total_count":10000,"filtered_count":250}`)
}))
defer server.Close()
page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues?columns=name", managementPageRequest{
Page: 2, PageSize: 100, Search: "orders & paid", Sort: "messages_ready", SortReverse: true,
})
if err != nil {
t.Fatal(err)
}
if requestCount != 1 {
t.Fatalf("expected one management request, got %d", requestCount)
}
if len(page.Items) != 1 || page.Page != 2 || page.PageSize != 100 || page.TotalCount != 250 || !page.HasMore {
t.Fatalf("unexpected page %#v", page)
}
}
func TestManagementGetPageFallsBackForPlainArrayResponses(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(writer, `[{"name":"alpha","messages_ready":1},{"name":"beta","messages_ready":9},{"name":"gamma","messages_ready":4}]`)
}))
defer server.Close()
page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues", managementPageRequest{
Page: 1, PageSize: 2, Search: "a", Sort: "messages_ready", SortReverse: true,
})
if err != nil {
t.Fatal(err)
}
if page.TotalCount != 3 || !page.HasMore || len(page.Items) != 2 {
t.Fatalf("unexpected page %#v", page)
}
first, _ := page.Items[0].(map[string]any)
second, _ := page.Items[1].(map[string]any)
if stringOrEmpty(jsonObject(first), "name") != "beta" || stringOrEmpty(jsonObject(second), "name") != "gamma" {
t.Fatalf("unexpected sorted items %#v", page.Items)
}
}
func TestManagementPageRequestValidation(t *testing.T) {
for _, params := range []jsonObject{
{"page": 0, "page_size": 100, "sort": "name"},
{"page": 1, "page_size": managementMaxPageSize + 1, "sort": "name"},
{"page": 1},
{"page": 1, "page_size": 100, "sort": "unsupported"},
} {
if _, err := managementPageRequestFromParams(params, "name"); err == nil {
t.Fatalf("expected invalid page request %#v to fail", params)
}
}
if page, err := managementPageRequestFromParams(jsonObject{}, "name"); err != nil || page != nil {
t.Fatalf("legacy unpaged request changed: %#v, %v", page, err)
}
}
func TestListTopicsPagePreservesAllVhostIdentityWithoutDrainingPages(t *testing.T) {
requestCount := 0
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requestCount++
if request.URL.Path != "/api/queues" {
t.Fatalf("unexpected all-vhosts path %q", request.URL.Path)
}
writer.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(writer, `{"items":[{"name":"orders","vhost":"staging","durable":true}],"page":1,"page_size":100,"page_count":100,"total_count":10000,"filtered_count":10000}`)
}))
defer server.Close()
service := newServer()
service.cachedConnection = jsonObject{"management_url": server.URL}
listed, err := service.listTopics(jsonObject{
"all_vhosts": true, "page": 1, "page_size": 100, "sort": "name",
})
if err != nil {
t.Fatal(err)
}
result := listed.(jsonObject)
topics := result["topics"].([]jsonObject)
if requestCount != 1 || len(topics) != 1 || stringOrEmpty(topics[0], "vhost") != "staging" {
t.Fatalf("unexpected paged topics %#v after %d requests", result, requestCount)
}
if intOrDefault(result, "totalCount", 0) != 10000 || !boolOrDefault(result, "hasMore", false) {
t.Fatalf("unexpected page metadata %#v", result)
}
}
func TestListExchangesPageSearchIncludesDisplayedDefaultExchange(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
query := request.URL.Query()
if query.Get("use_regex") != "true" || !strings.Contains(query.Get("name"), "^$") {
t.Fatalf("default exchange search was not translated to a regex: %q", request.URL.RawQuery)
}
writer.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(writer, `{"items":[{"name":"","type":"direct","durable":true}],"page":1,"page_size":100,"page_count":1,"total_count":8,"filtered_count":1}`)
}))
defer server.Close()
service := newServer()
service.cachedConnection = jsonObject{"management_url": server.URL}
listed, err := service.listExchanges(jsonObject{
"virtual_host": "/", "page": 1, "page_size": 100, "search": "default", "sort": "name",
})
if err != nil {
t.Fatal(err)
}
result := listed.(jsonObject)
exchanges := result["exchanges"].([]jsonObject)
if len(exchanges) != 1 || stringOrEmpty(exchanges[0], "name") != "" || intOrDefault(result, "totalCount", 0) != 1 {
t.Fatalf("unexpected exchange page %#v", result)
}
}
func TestManagementRequestFailsOverConnectionErrors(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.2:0")
if err != nil {
+64 -10
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"os"
"regexp"
"sort"
"strings"
"unicode/utf8"
@@ -30,7 +31,22 @@ func (s *server) listTopics(params jsonObject) (any, error) {
return nil, err
}
allVhosts := allVhostsRequested(params)
queues, err := managementGetAll(connection, managementListPath(params, connection, "queues"))
pageRequest, err := managementPageRequestFromParams(params, "name", "messages_ready")
if err != nil {
return nil, err
}
var queues []any
var pageResult *managementPageResult
if pageRequest == nil {
queues, err = managementGetAll(connection, managementListPath(params, connection, "queues"))
} else {
page, pageErr := managementGetPage(connection, managementListPath(params, connection, "queues"), *pageRequest)
if pageErr != nil {
return nil, pageErr
}
queues = page.Items
pageResult = &page
}
if err != nil {
return nil, err
}
@@ -46,10 +62,12 @@ func (s *server) listTopics(params jsonObject) (any, error) {
}
topics = append(topics, info)
}
sort.SliceStable(topics, func(left, right int) bool {
return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name")
})
return jsonObject{"topics": topics}, nil
if pageRequest == nil {
sort.SliceStable(topics, func(left, right int) bool {
return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name")
})
}
return managementListResult("topics", topics, pageResult), nil
}
func topicInfoFromJSON(queue jsonObject) jsonObject {
@@ -382,7 +400,29 @@ func (s *server) listExchanges(params jsonObject) (any, error) {
return nil, err
}
allVhosts := allVhostsRequested(params)
exchanges, err := managementGetAll(connection, managementListPath(params, connection, "exchanges"))
pageRequest, err := managementPageRequestFromParams(params, "name")
if err != nil {
return nil, err
}
if pageRequest != nil && pageRequest.Search != "" && strings.Contains(strings.ToLower("(AMQP default)"), strings.ToLower(pageRequest.Search)) {
// The UI displays the empty-name default exchange as "(AMQP default)".
// Include it in the broker-side name search whenever that display label
// matches, while retaining ordinary substring matches.
pageRequest.Search = "(" + regexp.QuoteMeta(pageRequest.Search) + "|^$)"
pageRequest.UseRegex = true
}
var exchanges []any
var pageResult *managementPageResult
if pageRequest == nil {
exchanges, err = managementGetAll(connection, managementListPath(params, connection, "exchanges"))
} else {
page, pageErr := managementGetPage(connection, managementListPath(params, connection, "exchanges"), *pageRequest)
if pageErr != nil {
return nil, pageErr
}
exchanges = page.Items
pageResult = &page
}
if err != nil {
return nil, err
}
@@ -398,10 +438,24 @@ func (s *server) listExchanges(params jsonObject) (any, error) {
}
result = append(result, info)
}
sort.SliceStable(result, func(left, right int) bool {
return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name")
})
return jsonObject{"exchanges": result}, nil
if pageRequest == nil {
sort.SliceStable(result, func(left, right int) bool {
return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name")
})
}
return managementListResult("exchanges", result, pageResult), nil
}
func managementListResult(key string, items []jsonObject, page *managementPageResult) jsonObject {
result := jsonObject{key: items}
if page == nil {
return result
}
result["page"] = page.Page
result["pageSize"] = page.PageSize
result["totalCount"] = page.TotalCount
result["hasMore"] = page.HasMore
return result
}
func exchangeInfoFromJSON(exchange jsonObject) jsonObject {