fix(rabbitmq): add bounded paging for queue and exchange management lists
Closes #8879
This commit is contained in:
1 parent
33f212a452
commit
725b1ee252
22 files changed
+1445
-67
No files matched your search
@@ -11,6 +11,9 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -21,10 +24,28 @@ const (
|
||||
defaultManagementTLSPort = 15671
|
||||
defaultAMQPTLSPort = 5671
|
||||
managementPageSize = 100
|
||||
managementMaxPageSize = 500
|
||||
managementConnectTimeout = 10 * time.Second
|
||||
managementRequestTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
type managementPageRequest struct {
|
||||
Page int
|
||||
PageSize int
|
||||
Search string
|
||||
UseRegex bool
|
||||
Sort string
|
||||
SortReverse bool
|
||||
}
|
||||
|
||||
type managementPageResult struct {
|
||||
Items []any
|
||||
Page int
|
||||
PageSize int
|
||||
TotalCount int
|
||||
HasMore bool
|
||||
}
|
||||
|
||||
type managementStatusError struct {
|
||||
status int
|
||||
method string
|
||||
@@ -172,6 +193,204 @@ func managementGetAll(connection jsonObject, path string) ([]any, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// managementGetPage requests exactly one RabbitMQ management API page. Queue
|
||||
// and exchange endpoints in RabbitMQ 3.8.x return a pagination envelope when
|
||||
// page/page_size are present. The plain-array fallback keeps the agent usable
|
||||
// with endpoints or older brokers that ignore those parameters, while still
|
||||
// presenting the same explicit page contract to callers.
|
||||
func managementGetPage(connection jsonObject, path string, request managementPageRequest) (managementPageResult, error) {
|
||||
if err := validateManagementPageRequest(request); err != nil {
|
||||
return managementPageResult{}, err
|
||||
}
|
||||
query := url.Values{}
|
||||
query.Set("page", strconv.Itoa(request.Page))
|
||||
query.Set("page_size", strconv.Itoa(request.PageSize))
|
||||
if request.Search != "" {
|
||||
query.Set("name", request.Search)
|
||||
query.Set("use_regex", strconv.FormatBool(request.UseRegex))
|
||||
}
|
||||
if request.Sort != "" {
|
||||
query.Set("sort", request.Sort)
|
||||
}
|
||||
if request.SortReverse {
|
||||
query.Set("sort_reverse", "true")
|
||||
}
|
||||
|
||||
response, err := managementGet(connection, appendManagementQuery(path, query))
|
||||
if err != nil {
|
||||
return managementPageResult{}, err
|
||||
}
|
||||
switch typed := response.(type) {
|
||||
case []any:
|
||||
return paginateManagementArray(typed, request)
|
||||
case map[string]any:
|
||||
object := jsonObject(typed)
|
||||
items, exists := typed["items"]
|
||||
if !exists {
|
||||
return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path)
|
||||
}
|
||||
array, ok := items.([]any)
|
||||
if !ok {
|
||||
return managementPageResult{}, fmt.Errorf("Unexpected management API items for list endpoint %s", path)
|
||||
}
|
||||
page := intOrDefault(object, "page", request.Page)
|
||||
pageSize := intOrDefault(object, "page_size", request.PageSize)
|
||||
totalCount := integerOrNull(object, "filtered_count")
|
||||
if totalCount == nil {
|
||||
totalCount = integerOrNull(object, "total_count")
|
||||
}
|
||||
if totalCount == nil {
|
||||
inferred := (page-1)*pageSize + len(array)
|
||||
totalCount = &inferred
|
||||
}
|
||||
pageCount := integerOrNull(object, "page_count")
|
||||
hasMore := pageCount != nil && page < *pageCount
|
||||
if pageCount == nil {
|
||||
hasMore = page*pageSize < *totalCount
|
||||
}
|
||||
return managementPageResult{
|
||||
Items: array,
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
TotalCount: *totalCount,
|
||||
HasMore: hasMore,
|
||||
}, nil
|
||||
default:
|
||||
return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func managementPageRequestFromParams(params jsonObject, allowedSorts ...string) (*managementPageRequest, error) {
|
||||
page := integerOrNull(params, "page")
|
||||
pageSize := integerOrNull(params, "page_size")
|
||||
if page == nil && pageSize == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if page == nil || pageSize == nil {
|
||||
return nil, errors.New("page and page_size must be provided together")
|
||||
}
|
||||
sortName := stringOrDefault(params, "sort", "name")
|
||||
allowed := false
|
||||
for _, candidate := range allowedSorts {
|
||||
if sortName == candidate {
|
||||
allowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allowed {
|
||||
return nil, fmt.Errorf("Unsupported list sort %q", sortName)
|
||||
}
|
||||
request := managementPageRequest{
|
||||
Page: *page,
|
||||
PageSize: *pageSize,
|
||||
Search: strings.TrimSpace(stringOrEmpty(params, "search")),
|
||||
Sort: sortName,
|
||||
SortReverse: boolOrDefault(params, "sort_reverse", false),
|
||||
}
|
||||
if err := validateManagementPageRequest(request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &request, nil
|
||||
}
|
||||
|
||||
func validateManagementPageRequest(request managementPageRequest) error {
|
||||
if request.Page < 1 {
|
||||
return errors.New("page must be at least 1")
|
||||
}
|
||||
if request.PageSize < 1 || request.PageSize > managementMaxPageSize {
|
||||
return fmt.Errorf("page_size must be between 1 and %d", managementMaxPageSize)
|
||||
}
|
||||
maxInt := int(^uint(0) >> 1)
|
||||
if request.Page-1 > maxInt/request.PageSize {
|
||||
return errors.New("page is too large")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendManagementQuery(path string, values url.Values) string {
|
||||
separator := "?"
|
||||
if strings.Contains(path, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
return path + separator + values.Encode()
|
||||
}
|
||||
|
||||
func paginateManagementArray(items []any, request managementPageRequest) (managementPageResult, error) {
|
||||
filtered := make([]any, 0, len(items))
|
||||
var expression *regexp.Regexp
|
||||
if request.Search != "" && request.UseRegex {
|
||||
compiled, err := regexp.Compile("(?i)" + request.Search)
|
||||
if err != nil {
|
||||
return managementPageResult{}, fmt.Errorf("Invalid management list search: %w", err)
|
||||
}
|
||||
expression = compiled
|
||||
}
|
||||
search := strings.ToLower(request.Search)
|
||||
for _, item := range items {
|
||||
if request.Search != "" {
|
||||
object, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := stringOrEmpty(jsonObject(object), "name")
|
||||
if expression != nil {
|
||||
if !expression.MatchString(name) {
|
||||
continue
|
||||
}
|
||||
} else if !strings.Contains(strings.ToLower(name), search) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
sort.SliceStable(filtered, func(left, right int) bool {
|
||||
leftObject, _ := filtered[left].(map[string]any)
|
||||
rightObject, _ := filtered[right].(map[string]any)
|
||||
leftJSON := jsonObject(leftObject)
|
||||
rightJSON := jsonObject(rightObject)
|
||||
comparison := 0
|
||||
if request.Sort == "messages_ready" {
|
||||
leftCount := longOrDefault(leftJSON, "messages_ready", 0)
|
||||
rightCount := longOrDefault(rightJSON, "messages_ready", 0)
|
||||
if leftCount < rightCount {
|
||||
comparison = -1
|
||||
} else if leftCount > rightCount {
|
||||
comparison = 1
|
||||
}
|
||||
}
|
||||
if comparison == 0 {
|
||||
comparison = strings.Compare(stringOrEmpty(leftJSON, "name"), stringOrEmpty(rightJSON, "name"))
|
||||
}
|
||||
if request.SortReverse {
|
||||
comparison = -comparison
|
||||
}
|
||||
return comparison < 0
|
||||
})
|
||||
|
||||
totalCount := len(filtered)
|
||||
start := (request.Page - 1) * request.PageSize
|
||||
if start >= totalCount {
|
||||
return managementPageResult{
|
||||
Items: []any{},
|
||||
Page: request.Page,
|
||||
PageSize: request.PageSize,
|
||||
TotalCount: totalCount,
|
||||
HasMore: false,
|
||||
}, nil
|
||||
}
|
||||
end := start + request.PageSize
|
||||
if end > totalCount {
|
||||
end = totalCount
|
||||
}
|
||||
return managementPageResult{
|
||||
Items: filtered[start:end],
|
||||
Page: request.Page,
|
||||
PageSize: request.PageSize,
|
||||
TotalCount: totalCount,
|
||||
HasMore: end < totalCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func managementBaseURLs(connection jsonObject) ([]string, error) {
|
||||
if explicit := stringOrNull(connection, "management_url"); explicit != nil && strings.TrimSpace(*explicit) != "" {
|
||||
return []string{normalizeManagementURL(*explicit)}, nil
|
||||
|
||||
@@ -135,6 +135,134 @@ func TestManagementGetAllPlainArray(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagementGetPageRequestsOnlyOneRabbitMqPage(t *testing.T) {
|
||||
requestCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
requestCount++
|
||||
query := request.URL.Query()
|
||||
if query.Get("page") != "2" || query.Get("page_size") != "100" {
|
||||
t.Fatalf("unexpected pagination query %q", request.URL.RawQuery)
|
||||
}
|
||||
if query.Get("name") != "orders & paid" || query.Get("use_regex") != "false" {
|
||||
t.Fatalf("unexpected search query %q", request.URL.RawQuery)
|
||||
}
|
||||
if query.Get("sort") != "messages_ready" || query.Get("sort_reverse") != "true" {
|
||||
t.Fatalf("unexpected sort query %q", request.URL.RawQuery)
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(writer, `{"items":[{"name":"orders.paid"}],"page":2,"page_size":100,"page_count":3,"total_count":10000,"filtered_count":250}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues?columns=name", managementPageRequest{
|
||||
Page: 2, PageSize: 100, Search: "orders & paid", Sort: "messages_ready", SortReverse: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if requestCount != 1 {
|
||||
t.Fatalf("expected one management request, got %d", requestCount)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Page != 2 || page.PageSize != 100 || page.TotalCount != 250 || !page.HasMore {
|
||||
t.Fatalf("unexpected page %#v", page)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagementGetPageFallsBackForPlainArrayResponses(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(writer, `[{"name":"alpha","messages_ready":1},{"name":"beta","messages_ready":9},{"name":"gamma","messages_ready":4}]`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues", managementPageRequest{
|
||||
Page: 1, PageSize: 2, Search: "a", Sort: "messages_ready", SortReverse: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if page.TotalCount != 3 || !page.HasMore || len(page.Items) != 2 {
|
||||
t.Fatalf("unexpected page %#v", page)
|
||||
}
|
||||
first, _ := page.Items[0].(map[string]any)
|
||||
second, _ := page.Items[1].(map[string]any)
|
||||
if stringOrEmpty(jsonObject(first), "name") != "beta" || stringOrEmpty(jsonObject(second), "name") != "gamma" {
|
||||
t.Fatalf("unexpected sorted items %#v", page.Items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagementPageRequestValidation(t *testing.T) {
|
||||
for _, params := range []jsonObject{
|
||||
{"page": 0, "page_size": 100, "sort": "name"},
|
||||
{"page": 1, "page_size": managementMaxPageSize + 1, "sort": "name"},
|
||||
{"page": 1},
|
||||
{"page": 1, "page_size": 100, "sort": "unsupported"},
|
||||
} {
|
||||
if _, err := managementPageRequestFromParams(params, "name"); err == nil {
|
||||
t.Fatalf("expected invalid page request %#v to fail", params)
|
||||
}
|
||||
}
|
||||
if page, err := managementPageRequestFromParams(jsonObject{}, "name"); err != nil || page != nil {
|
||||
t.Fatalf("legacy unpaged request changed: %#v, %v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTopicsPagePreservesAllVhostIdentityWithoutDrainingPages(t *testing.T) {
|
||||
requestCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
requestCount++
|
||||
if request.URL.Path != "/api/queues" {
|
||||
t.Fatalf("unexpected all-vhosts path %q", request.URL.Path)
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(writer, `{"items":[{"name":"orders","vhost":"staging","durable":true}],"page":1,"page_size":100,"page_count":100,"total_count":10000,"filtered_count":10000}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
service := newServer()
|
||||
service.cachedConnection = jsonObject{"management_url": server.URL}
|
||||
|
||||
listed, err := service.listTopics(jsonObject{
|
||||
"all_vhosts": true, "page": 1, "page_size": 100, "sort": "name",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := listed.(jsonObject)
|
||||
topics := result["topics"].([]jsonObject)
|
||||
if requestCount != 1 || len(topics) != 1 || stringOrEmpty(topics[0], "vhost") != "staging" {
|
||||
t.Fatalf("unexpected paged topics %#v after %d requests", result, requestCount)
|
||||
}
|
||||
if intOrDefault(result, "totalCount", 0) != 10000 || !boolOrDefault(result, "hasMore", false) {
|
||||
t.Fatalf("unexpected page metadata %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListExchangesPageSearchIncludesDisplayedDefaultExchange(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
query := request.URL.Query()
|
||||
if query.Get("use_regex") != "true" || !strings.Contains(query.Get("name"), "^$") {
|
||||
t.Fatalf("default exchange search was not translated to a regex: %q", request.URL.RawQuery)
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(writer, `{"items":[{"name":"","type":"direct","durable":true}],"page":1,"page_size":100,"page_count":1,"total_count":8,"filtered_count":1}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
service := newServer()
|
||||
service.cachedConnection = jsonObject{"management_url": server.URL}
|
||||
|
||||
listed, err := service.listExchanges(jsonObject{
|
||||
"virtual_host": "/", "page": 1, "page_size": 100, "search": "default", "sort": "name",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := listed.(jsonObject)
|
||||
exchanges := result["exchanges"].([]jsonObject)
|
||||
if len(exchanges) != 1 || stringOrEmpty(exchanges[0], "name") != "" || intOrDefault(result, "totalCount", 0) != 1 {
|
||||
t.Fatalf("unexpected exchange page %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagementRequestFailsOverConnectionErrors(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.2:0")
|
||||
if err != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
@@ -30,7 +31,22 @@ func (s *server) listTopics(params jsonObject) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
allVhosts := allVhostsRequested(params)
|
||||
queues, err := managementGetAll(connection, managementListPath(params, connection, "queues"))
|
||||
pageRequest, err := managementPageRequestFromParams(params, "name", "messages_ready")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var queues []any
|
||||
var pageResult *managementPageResult
|
||||
if pageRequest == nil {
|
||||
queues, err = managementGetAll(connection, managementListPath(params, connection, "queues"))
|
||||
} else {
|
||||
page, pageErr := managementGetPage(connection, managementListPath(params, connection, "queues"), *pageRequest)
|
||||
if pageErr != nil {
|
||||
return nil, pageErr
|
||||
}
|
||||
queues = page.Items
|
||||
pageResult = &page
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -46,10 +62,12 @@ func (s *server) listTopics(params jsonObject) (any, error) {
|
||||
}
|
||||
topics = append(topics, info)
|
||||
}
|
||||
sort.SliceStable(topics, func(left, right int) bool {
|
||||
return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name")
|
||||
})
|
||||
return jsonObject{"topics": topics}, nil
|
||||
if pageRequest == nil {
|
||||
sort.SliceStable(topics, func(left, right int) bool {
|
||||
return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name")
|
||||
})
|
||||
}
|
||||
return managementListResult("topics", topics, pageResult), nil
|
||||
}
|
||||
|
||||
func topicInfoFromJSON(queue jsonObject) jsonObject {
|
||||
@@ -382,7 +400,29 @@ func (s *server) listExchanges(params jsonObject) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
allVhosts := allVhostsRequested(params)
|
||||
exchanges, err := managementGetAll(connection, managementListPath(params, connection, "exchanges"))
|
||||
pageRequest, err := managementPageRequestFromParams(params, "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pageRequest != nil && pageRequest.Search != "" && strings.Contains(strings.ToLower("(AMQP default)"), strings.ToLower(pageRequest.Search)) {
|
||||
// The UI displays the empty-name default exchange as "(AMQP default)".
|
||||
// Include it in the broker-side name search whenever that display label
|
||||
// matches, while retaining ordinary substring matches.
|
||||
pageRequest.Search = "(" + regexp.QuoteMeta(pageRequest.Search) + "|^$)"
|
||||
pageRequest.UseRegex = true
|
||||
}
|
||||
var exchanges []any
|
||||
var pageResult *managementPageResult
|
||||
if pageRequest == nil {
|
||||
exchanges, err = managementGetAll(connection, managementListPath(params, connection, "exchanges"))
|
||||
} else {
|
||||
page, pageErr := managementGetPage(connection, managementListPath(params, connection, "exchanges"), *pageRequest)
|
||||
if pageErr != nil {
|
||||
return nil, pageErr
|
||||
}
|
||||
exchanges = page.Items
|
||||
pageResult = &page
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -398,10 +438,24 @@ func (s *server) listExchanges(params jsonObject) (any, error) {
|
||||
}
|
||||
result = append(result, info)
|
||||
}
|
||||
sort.SliceStable(result, func(left, right int) bool {
|
||||
return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name")
|
||||
})
|
||||
return jsonObject{"exchanges": result}, nil
|
||||
if pageRequest == nil {
|
||||
sort.SliceStable(result, func(left, right int) bool {
|
||||
return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name")
|
||||
})
|
||||
}
|
||||
return managementListResult("exchanges", result, pageResult), nil
|
||||
}
|
||||
|
||||
func managementListResult(key string, items []jsonObject, page *managementPageResult) jsonObject {
|
||||
result := jsonObject{key: items}
|
||||
if page == nil {
|
||||
return result
|
||||
}
|
||||
result["page"] = page.Page
|
||||
result["pageSize"] = page.PageSize
|
||||
result["totalCount"] = page.TotalCount
|
||||
result["hasMore"] = page.HasMore
|
||||
return result
|
||||
}
|
||||
|
||||
func exchangeInfoFromJSON(exchange jsonObject) jsonObject {
|
||||
|
||||
Reference in new issue
Block a user