From d0031e41067a3bbf710b6834e8ff065cdbb531f8 Mon Sep 17 00:00:00 2001 From: wweir Date: Fri, 18 Sep 2026 22:37:03 +0800 Subject: [PATCH] fix(sower): keep the soft heap limit above the live rule heap The default 128 MiB soft memory limit sits below the ~105 MB live heap the full pexcn rule sets build, so the GC goal gets pinned to the live heap and every allocation triggers a full-heap mark: measured 39% CPU on an otherwise idle gateway with a GC cycle every 300-500 ms. Raise the default to 512 MiB, resolve the limit in a testable helper that covers GOMEMLIMIT/override/disable precedence, and declare the same value in the packaged service plus the example configs so a deployment can tune it without a rebuild. --- .github/sower.service | 5 +++ .github/sower.toml | 8 +++++ cmd/sower/main.go | 73 +++++++++++++++++++++++++++--------------- cmd/sower/main_test.go | 30 +++++++++++++++++ config/sower.toml | 6 ++++ 5 files changed, 97 insertions(+), 25 deletions(-) diff --git a/.github/sower.service b/.github/sower.service index a464b15..5616d74 100644 --- a/.github/sower.service +++ b/.github/sower.service @@ -5,6 +5,11 @@ After=network.target [Service] Type=simple WorkingDirectory=/etc/sower +# Keep the soft heap limit well above the ~105MB live rule heap: a limit at +# that level pins the GC goal to the live heap and burns CPU on every +# allocation. Matches the binary default and is declared here so a deployment +# can tune it without rebuilding (0 disables the soft limit). +Environment=SOWER_MEMORY_LIMIT_MB=512 ExecStart=/usr/local/bin/sower -c /etc/sower/sower.toml Restart=on-failure RestartSec=1min diff --git a/.github/sower.toml b/.github/sower.toml index f43a49e..01a748a 100644 --- a/.github/sower.toml +++ b/.github/sower.toml @@ -1,3 +1,11 @@ +# Sower client configuration packaged with releases. +# +# The binary caps its Go heap with a 512 MiB soft memory limit by default; that +# limit has to stay well above the ~105MB live rule heap, because a limit at +# that level pins the GC goal to the live heap and burns CPU on every +# allocation. Override it with SOWER_MEMORY_LIMIT_MB in the service environment +# (0 disables the soft limit), not with a TOML field. + # Remote upstream connection. [remote] type = "sower" diff --git a/cmd/sower/main.go b/cmd/sower/main.go index 59a352d..15a8ec7 100644 --- a/cmd/sower/main.go +++ b/cmd/sower/main.go @@ -43,8 +43,11 @@ var ( ) // defaultMemoryLimitMiB bounds the Go heap unless GOMEMLIMIT or -// SOWER_MEMORY_LIMIT_MB overrides it; see init for the rationale. -const defaultMemoryLimitMiB = 128 +// SOWER_MEMORY_LIMIT_MB overrides it. It must stay well above the live rule +// heap: the full pexcn rule sets keep ~105MB alive, and a limit at that level +// pins the GC goal to the live heap, so every allocation triggers a full-heap +// mark (128MiB burned ~39% CPU on an otherwise idle gateway). +const defaultMemoryLimitMiB = 512 // maxMemoryLimitMiB caps the SOWER_MEMORY_LIMIT_MB override so the MiB // shift cannot overflow int64. @@ -76,6 +79,34 @@ func logConfigLoadError(err error, cfg config.SowerConfig) { slog.Error("load config", "error", err, "remote_addr", cfg.Remote.Addr) } +// memoryLimitMiB resolves the soft heap limit for this process. +// +// -1 => the soft limit is explicitly disabled (apply math.MaxInt64) +// 0 => leave the limit alone (an explicit GOMEMLIMIT is in charge) +// >0 => apply that many MiB +func memoryLimitMiB(getenv func(string) string) int64 { + if getenv("GOMEMLIMIT") != "" { + return 0 + } + + v := getenv("SOWER_MEMORY_LIMIT_MB") + if v == "" { + return defaultMemoryLimitMiB + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return defaultMemoryLimitMiB + } + switch { + case n == 0: + return -1 + case n > 0 && n <= maxMemoryLimitMiB: + return n + default: + return defaultMemoryLimitMiB + } +} + func init() { fi, _ := os.Stdout.Stat() noColor := (fi.Mode() & os.ModeCharDevice) == 0 @@ -97,31 +128,23 @@ func init() { os.Exit(1) } - // Bound the Go heap with a soft memory limit. The default rule sets - // (adlist/chinalist/gfwlist) build ~40MB of suffix trees, and GOGC's - // 2x target on top of that pushes resident memory toward 250MB on a - // gateway that mostly idles. The soft limit makes GC reclaim eagerly - // and return memory to the OS. + // Bound the Go heap with a soft memory limit so a mostly idle gateway + // returns memory to the OS instead of holding GOGC's 2x headroom. + // + // The limit must stay well above the live rule heap: the full pexcn rule + // sets keep ~105MB alive, and a limit at that level pins the GC goal to + // the live heap, so every allocation triggers a full-heap mark. 512MiB + // leaves the usual headroom while still capping resident memory. // // Precedence: an explicit standard GOMEMLIMIT wins; otherwise the - // default 128MiB applies unless SOWER_MEMORY_LIMIT_MB overrides it - // (a positive MiB value, or 0 to disable the soft limit). Malformed or - // negative values leave the default in place. - if os.Getenv("GOMEMLIMIT") == "" { - var limit int64 = defaultMemoryLimitMiB - if v := os.Getenv("SOWER_MEMORY_LIMIT_MB"); v != "" { - if n, err := strconv.ParseInt(v, 10, 64); err == nil { - switch { - case n == 0: - debug.SetMemoryLimit(math.MaxInt64) // disable - case n > 0 && n <= maxMemoryLimitMiB: - limit = n - } - } - } - if limit > 0 { - debug.SetMemoryLimit(limit << 20) - } + // default applies unless SOWER_MEMORY_LIMIT_MB overrides it (a positive + // MiB value, or 0 to disable the soft limit). Malformed or negative + // values leave the default in place. + switch limit := memoryLimitMiB(os.Getenv); { + case limit < 0: + debug.SetMemoryLimit(math.MaxInt64) // disable + case limit > 0: + debug.SetMemoryLimit(limit << 20) } logLevel.Set(conf.LogLevel) diff --git a/cmd/sower/main_test.go b/cmd/sower/main_test.go index 9f8a5f7..e8d8dd8 100644 --- a/cmd/sower/main_test.go +++ b/cmd/sower/main_test.go @@ -263,3 +263,33 @@ func TestLogConfigLoadErrorRedactsRemotePassword(t *testing.T) { t.Fatalf("load-failure log missing error context: %s", out) } } + +func TestMemoryLimitMiB(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + env map[string]string + want int64 + }{ + {name: "default", want: defaultMemoryLimitMiB}, + {name: "explicit GOMEMLIMIT wins", env: map[string]string{"GOMEMLIMIT": "1GiB"}, want: 0}, + {name: "override", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": "256"}, want: 256}, + {name: "zero disables the soft limit", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": "0"}, want: -1}, + {name: "garbage keeps the default", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": "lots"}, want: defaultMemoryLimitMiB}, + {name: "negative keeps the default", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": "-5"}, want: defaultMemoryLimitMiB}, + {name: "above the cap keeps the default", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": strconv.FormatInt(maxMemoryLimitMiB+1, 10)}, want: defaultMemoryLimitMiB}, + {name: "at the cap", env: map[string]string{"SOWER_MEMORY_LIMIT_MB": strconv.FormatInt(maxMemoryLimitMiB, 10)}, want: maxMemoryLimitMiB}, + {name: "GOMEMLIMIT beats the override", env: map[string]string{"GOMEMLIMIT": "1GiB", "SOWER_MEMORY_LIMIT_MB": "256"}, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := memoryLimitMiB(func(k string) string { return tt.env[k] }) + if got != tt.want { + t.Fatalf("memoryLimitMiB(%v) = %d, want %d", tt.env, got, tt.want) + } + }) + } +} diff --git a/config/sower.toml b/config/sower.toml index 471acfd..7572a49 100644 --- a/config/sower.toml +++ b/config/sower.toml @@ -1,5 +1,11 @@ # Sower configuration example (TOML format) # This is the client configuration for the sower proxy +# +# The binary caps its Go heap with a 512 MiB soft memory limit by default; that +# limit has to stay well above the ~105MB live rule heap, because a limit at +# that level pins the GC goal to the live heap and burns CPU on every +# allocation. Override it with SOWER_MEMORY_LIMIT_MB in the service environment +# (0 disables the soft limit), not with a TOML field. # Logging configuration log_level = "info" # Log level: debug, info, warn, error