307 lines
8.8 KiB
Go
307 lines
8.8 KiB
Go
package gateway
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"linkport/internal/model"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Metrics count TCP acceptance separately from protocol authentication.
|
|
type Metrics struct {
|
|
TotalConnections int64 `json:"totalConnections"`
|
|
RXBytes int64 `json:"rxBytes"`
|
|
TXBytes int64 `json:"txBytes"`
|
|
RecentError string `json:"recentError,omitempty"`
|
|
RecentErrorAt time.Time `json:"recentErrorAt,omitempty"`
|
|
ConnectionAttempts int64 `json:"connectionAttempts"`
|
|
ConnectionRejected int64 `json:"connectionRejected"`
|
|
UnauthenticatedClosed int64 `json:"unauthenticatedClosed"`
|
|
AuthenticatedConnections int64 `json:"authenticatedConnections"`
|
|
AuthenticationSucceeded int64 `json:"authenticationSucceeded"`
|
|
AuthenticationFailed int64 `json:"authenticationFailed"`
|
|
ActiveConnections int `json:"activeConnections"`
|
|
ConnectionFailureRate float64 `json:"connectionFailureRate"`
|
|
AuthenticationFailureRate float64 `json:"authenticationFailureRate"`
|
|
MeanAuthenticationLatencyMS float64 `json:"meanAuthenticationLatencyMs"`
|
|
AuthenticationLatencyTotalMS float64 `json:"-"`
|
|
}
|
|
|
|
type HealthResult struct {
|
|
TargetID string `json:"targetId"`
|
|
AccountID string `json:"accountId"`
|
|
Protocol string `json:"protocol"`
|
|
Check string `json:"check"`
|
|
CheckedAt time.Time `json:"checkedAt"`
|
|
Healthy bool `json:"healthy"`
|
|
Attempts int `json:"attempts"`
|
|
LatencyMS float64 `json:"latencyMs"`
|
|
Stage string `json:"stage,omitempty"`
|
|
Result model.TestResult `json:"result"`
|
|
}
|
|
|
|
type SlowRequest struct {
|
|
GatewayID string `json:"gatewayId"`
|
|
TargetID string `json:"targetId"`
|
|
AccountID string `json:"accountId"`
|
|
model.Execution
|
|
}
|
|
|
|
func (m *Manager) SlowSnapshot() []SlowRequest {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
return append([]SlowRequest{}, m.slowRequests...)
|
|
}
|
|
|
|
func (m *Manager) recordExecution(p model.Proxy, e model.Execution) {
|
|
if e.DurationMS < 1000 {
|
|
return
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.slowRequests = append(m.slowRequests, SlowRequest{GatewayID: p.ID, TargetID: p.TargetID, AccountID: p.TargetAccountID, Execution: e})
|
|
if len(m.slowRequests) > 500 {
|
|
copy(m.slowRequests, m.slowRequests[len(m.slowRequests)-500:])
|
|
m.slowRequests = m.slowRequests[:500]
|
|
}
|
|
}
|
|
|
|
func (m *Manager) metric(id string) *Metrics {
|
|
v := m.metrics[id]
|
|
if v == nil {
|
|
v = &Metrics{}
|
|
m.metrics[id] = v
|
|
}
|
|
return v
|
|
}
|
|
|
|
func (m *Manager) Metrics(id string) Metrics {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
v := *m.metric(id)
|
|
if instance := m.instances[id]; instance != nil {
|
|
v.ActiveConnections = len(instance.connections)
|
|
}
|
|
if v.ConnectionAttempts > 0 {
|
|
v.ConnectionFailureRate = float64(v.ConnectionRejected+v.UnauthenticatedClosed) / float64(v.ConnectionAttempts)
|
|
}
|
|
if total := v.AuthenticationSucceeded + v.AuthenticationFailed; total > 0 {
|
|
v.AuthenticationFailureRate = float64(v.AuthenticationFailed) / float64(total)
|
|
}
|
|
if v.AuthenticatedConnections > 0 {
|
|
v.MeanAuthenticationLatencyMS = v.AuthenticationLatencyTotalMS / float64(v.AuthenticatedConnections)
|
|
}
|
|
return v
|
|
}
|
|
|
|
func (m *Manager) HealthSnapshot() (int, []HealthResult) {
|
|
m.mu.Lock()
|
|
interval := m.healthInterval
|
|
results := make([]HealthResult, 0, len(m.health))
|
|
for _, result := range m.health {
|
|
results = append(results, result)
|
|
}
|
|
m.mu.Unlock()
|
|
sort.Slice(results, func(i, j int) bool {
|
|
if results[i].TargetID != results[j].TargetID {
|
|
return results[i].TargetID < results[j].TargetID
|
|
}
|
|
return results[i].AccountID < results[j].AccountID
|
|
})
|
|
return interval, results
|
|
}
|
|
|
|
// CheckHealth retries only the protocol's fixed health probe, never live sessions.
|
|
func (m *Manager) CheckHealth(ctx context.Context, p model.Proxy, attempts int) (HealthResult, error) {
|
|
if attempts < 1 || attempts > 3 {
|
|
return HealthResult{}, errors.New("attempts must be between 1 and 3")
|
|
}
|
|
key := p.TargetID + "/" + p.TargetAccountID
|
|
m.mu.Lock()
|
|
if m.healthBusy[key] {
|
|
m.mu.Unlock()
|
|
return HealthResult{}, errors.New("health check already running for this account")
|
|
}
|
|
m.healthBusy[key] = true
|
|
m.mu.Unlock()
|
|
defer func() { m.mu.Lock(); delete(m.healthBusy, key); m.mu.Unlock() }()
|
|
ctx, cancel := context.WithTimeout(ctx, 25*time.Second)
|
|
defer cancel()
|
|
started := time.Now()
|
|
result := HealthResult{TargetID: p.TargetID, AccountID: p.TargetAccountID}
|
|
result.Protocol = p.Type
|
|
result.Check = protocolCheck(p.Type)
|
|
for attempt := 1; attempt <= attempts; attempt++ {
|
|
if err := ctx.Err(); err != nil {
|
|
result.Stage = FailureCategory(err)
|
|
break
|
|
}
|
|
result.Attempts = attempt
|
|
probe, err := m.Test(ctx, p)
|
|
if err == nil {
|
|
result.Healthy = true
|
|
result.Result = probe
|
|
result.Stage = "ok"
|
|
break
|
|
}
|
|
result.Stage = FailureCategory(err)
|
|
if attempt == attempts || (result.Stage != "connection_timeout" && result.Stage != "target_connect_refused" && result.Stage != "target_connect_timeout" && result.Stage != "target_connect_dns" && result.Stage != "target_connect_unreachable" && result.Stage != "target_connect_reset" && result.Stage != "target_connection_or_protocol" && result.Stage != "socks5_connection" && result.Stage != "ssh_forward_connection") {
|
|
break
|
|
}
|
|
timer := time.NewTimer(time.Duration(attempt) * 100 * time.Millisecond)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
result.CheckedAt = time.Now().UTC()
|
|
result.LatencyMS = float64(time.Since(started).Microseconds()) / 1000
|
|
// Stopping/reconfiguring a worker is not evidence that a target is unhealthy.
|
|
// Real probe deadlines still produce a failed health result.
|
|
if errors.Is(ctx.Err(), context.Canceled) {
|
|
return result, ctx.Err()
|
|
}
|
|
m.mu.Lock()
|
|
m.health[key] = result
|
|
m.mu.Unlock()
|
|
return result, nil
|
|
}
|
|
|
|
func protocolCheck(protocol string) string {
|
|
switch protocol {
|
|
case "mysql":
|
|
return "MySQL handshake + ping"
|
|
case "pgsql":
|
|
return "PostgreSQL startup + SELECT 1"
|
|
case "mongodb":
|
|
return "MongoDB hello/buildInfo"
|
|
case "redis":
|
|
return "Redis AUTH + PING"
|
|
case "rabbitmq":
|
|
return "AMQP connection + virtual host"
|
|
case "sqlserver":
|
|
return "TDS prelogin + LOGIN7"
|
|
case "http", "https":
|
|
return "HTTP HEAD + Basic Auth"
|
|
case "ssh":
|
|
return "SSH handshake + authentication"
|
|
case "sftp":
|
|
return "SSH + SFTP subsystem + working directory"
|
|
default:
|
|
return "protocol health probe"
|
|
}
|
|
}
|
|
|
|
// SetHealthInterval persists operator settings. Shutdown only stops the worker.
|
|
func (m *Manager) SetHealthInterval(seconds int) error {
|
|
return m.setHealthInterval(seconds, true)
|
|
}
|
|
func (m *Manager) ResumeHealthChecks() error {
|
|
seconds, err := m.store.HealthInterval()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return m.setHealthInterval(seconds, false)
|
|
}
|
|
func (m *Manager) setHealthInterval(seconds int, persist bool) error {
|
|
if seconds != 0 && (seconds < 10 || seconds > 86400) {
|
|
return errors.New("intervalSeconds must be 0 or between 10 and 86400")
|
|
}
|
|
m.healthMu.Lock()
|
|
defer m.healthMu.Unlock()
|
|
if persist {
|
|
if err := m.store.SaveHealthInterval(seconds); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if m.healthCancel != nil {
|
|
m.healthCancel()
|
|
<-m.healthDone
|
|
m.healthCancel = nil
|
|
}
|
|
m.mu.Lock()
|
|
m.healthInterval = seconds
|
|
m.mu.Unlock()
|
|
if seconds == 0 {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
m.healthCancel = cancel
|
|
m.healthDone = make(chan struct{})
|
|
done := m.healthDone
|
|
go func() {
|
|
defer close(done)
|
|
ticker := time.NewTicker(time.Duration(seconds) * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
m.checkAllAccounts(ctx)
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) checkAllAccounts(ctx context.Context) {
|
|
targets, err := m.store.Targets()
|
|
if err != nil {
|
|
return
|
|
}
|
|
active := map[string]bool{}
|
|
type job struct {
|
|
target model.Target
|
|
account model.TargetAccount
|
|
}
|
|
jobs := make(chan job)
|
|
var workers sync.WaitGroup
|
|
// Bound periodic network work without letting one slow account block the sweep.
|
|
for range 4 {
|
|
workers.Add(1)
|
|
go func() {
|
|
defer workers.Done()
|
|
for next := range jobs {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
p := next.target.Runtime()
|
|
p.TargetAccountID = next.account.ID
|
|
p = next.target.Apply(p)
|
|
m.CheckHealth(ctx, p, 1)
|
|
}
|
|
}()
|
|
}
|
|
sweep:
|
|
for _, target := range targets {
|
|
accounts := target.Accounts
|
|
if target.ValidAccountSelection("") {
|
|
accounts = []model.TargetAccount{{}}
|
|
}
|
|
for _, account := range accounts {
|
|
active[target.ID+"/"+account.ID] = true
|
|
select {
|
|
case <-ctx.Done():
|
|
break sweep
|
|
case jobs <- job{target, account}:
|
|
}
|
|
}
|
|
}
|
|
close(jobs)
|
|
workers.Wait()
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
for key := range m.health {
|
|
if !active[key] {
|
|
delete(m.health, key)
|
|
}
|
|
}
|
|
}
|