From acd32b85ee6e38460b623019326781233c9bca3c Mon Sep 17 00:00:00 2001 From: 0xJacky Date: Thu, 13 Aug 2026 10:19:04 +0800 Subject: [PATCH] feat(cluster): explain why a node is unreachable --- api/cluster/node.go | 26 +++-- app/src/api/node.ts | 3 + app/src/pinia/moudule/nodeAvailability.ts | 3 + app/src/views/node/nodeColumns.tsx | 78 ++++++++++++- e2e/tests/node-connection-error.spec.ts | 47 ++++++++ internal/analytic/node.go | 7 ++ internal/analytic/node_record.go | 61 ++++++++-- internal/analytic/node_record_test.go | 65 ++++++++++- internal/analytic/node_time_skew_test.go | 129 ++++++++++++++++++++++ 9 files changed, 396 insertions(+), 23 deletions(-) create mode 100644 e2e/tests/node-connection-error.spec.ts create mode 100644 internal/analytic/node_time_skew_test.go diff --git a/api/cluster/node.go b/api/cluster/node.go index 1e51e093..a90939db 100644 --- a/api/cluster/node.go +++ b/api/cluster/node.go @@ -31,16 +31,19 @@ type nodeMutationRequest struct { } type nodeResponse struct { - ID uint64 `json:"id"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Name string `json:"name"` - URL string `json:"url"` - Enabled bool `json:"enabled"` - AuthMethod string `json:"auth_method"` - HasCredential bool `json:"has_credential"` - CredentialStatus string `json:"credential_status"` - LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` + ID uint64 `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Name string `json:"name"` + URL string `json:"url"` + Enabled bool `json:"enabled"` + AuthMethod string `json:"auth_method"` + HasCredential bool `json:"has_credential"` + CredentialStatus string `json:"credential_status"` + LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` + ConnectionError string `json:"connection_error,omitempty"` + ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"` + ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"` // LegacySecret only ever carries the redaction sentinel, which tells the // edit form a secret is stored without putting it in a list response. LegacySecret string `json:"legacy_secret,omitempty"` @@ -68,6 +71,9 @@ func newNodeResponse(node *model.Node) nodeResponse { if analyticNode != nil { response.NodeStat = analyticNode.NodeStat response.NodeInfo = analyticNode.NodeInfo + response.ConnectionError = analyticNode.ConnectionError + response.ConnectionErrorCode = analyticNode.ConnectionErrorCode + response.ConnectionErrorAt = analyticNode.ConnectionErrorAt } return response } diff --git a/app/src/api/node.ts b/app/src/api/node.ts index e16177d0..a35a01d5 100644 --- a/app/src/api/node.ts +++ b/app/src/api/node.ts @@ -14,6 +14,9 @@ export interface Node extends ModelBase { credential_status: 'active' | 'unpaired' | 'rotating' | 'revoked' last_credential_use_at?: string response_at?: string + connection_error?: string + connection_error_code?: 'clock_skew' + connection_error_at?: string } export interface NodeStatus { diff --git a/app/src/pinia/moudule/nodeAvailability.ts b/app/src/pinia/moudule/nodeAvailability.ts index ade8cfed..ce5c515f 100644 --- a/app/src/pinia/moudule/nodeAvailability.ts +++ b/app/src/pinia/moudule/nodeAvailability.ts @@ -103,6 +103,9 @@ export const useNodeAvailabilityStore = defineStore('nodeAvailability', () => { auth_method: node.auth_method, has_credential: node.has_credential, credential_status: node.credential_status, + connection_error: node.connection_error, + connection_error_code: node.connection_error_code, + connection_error_at: node.connection_error_at, enabled: true, } }) diff --git a/app/src/views/node/nodeColumns.tsx b/app/src/views/node/nodeColumns.tsx index 7cd17dce..cec97bbe 100644 --- a/app/src/views/node/nodeColumns.tsx +++ b/app/src/views/node/nodeColumns.tsx @@ -1,8 +1,9 @@ import type { CustomRenderArgs, StdTableColumn } from '@uozi-admin/curd' import type { JSX } from 'vue/jsx-runtime' import type { Node } from '@/api/node' +import { ExclamationCircleOutlined, InfoCircleOutlined } from '@ant-design/icons-vue' import { datetimeRender } from '@uozi-admin/curd' -import { Badge, InputPassword, Tag } from 'ant-design-vue' +import { Badge, InputPassword, Popover, Tag } from 'ant-design-vue' import { h } from 'vue' import nodeApi from '@/api/node' import { SensitiveInput } from '@/components/SensitiveString' @@ -15,6 +16,76 @@ const unhealthyCredentialMap: Record string revoked: { color: 'red', text: () => $gettext('Revoked') }, } +function renderConnectionErrorContent(record: Node) { + if (record.connection_error_code !== 'clock_skew') { + return ( +
+

+ {$gettext('Check the node URL, network, TLS certificate, and authentication settings.')} +

+
+ + {$gettext('Technical details')} + + 
+            {record.connection_error}
+
+
+
+ ) + } + + return ( +
+

+ {$gettext('The controller time is earlier than the node certificate validity start time. TLS stops the connection before node authentication.')} +

+
    +
  1. {$gettext('Synchronize the host system time on both the controller and the node.')}
    2. +
  2. {$gettext('If Nginx UI runs in Docker, correct the Docker host clock instead of the container clock.')}
    3. +
+

+ {$gettext('On Linux hosts, check and enable network time synchronization:')} +

+ 
+        {'timedatectl status\nsudo timedatectl set-ntp true'}
+
+

+ {$gettext('The node will reconnect automatically after the clocks are synchronized.')} +

+
+ + {$gettext('Technical details')} + + 
+          {record.connection_error}
+
+
+
+ ) +} + +function renderConnectionError(record: Node) { + const isClockSkew = record.connection_error_code === 'clock_skew' + const title = isClockSkew + ? $gettext('System clocks are out of sync') + : $gettext('Node connection failed') + + return h(Popover, { + content: renderConnectionErrorContent(record), + placement: 'rightTop', + title: h('div', { class: 'flex items-center gap-2' }, [ + h(ExclamationCircleOutlined, { class: isClockSkew ? 'text-orange-500' : 'text-red-500' }), + h('span', title), + ]), + trigger: ['hover', 'focus', 'click'], + }, () => h('button', { + 'type': 'button', + 'aria-label': title, + 'class': 'ml-1 inline-flex cursor-help items-center border-0 bg-transparent p-0 text-red-500', + }, h(InfoCircleOutlined))) +} + const columns: StdTableColumn[] = [{ title: () => $gettext('Name'), dataIndex: 'name', @@ -104,7 +175,10 @@ const columns: StdTableColumn[] = [{ template.push({$gettext('Disabled')}) } - return h('div', template) + if (args.record.connection_error) + template.push(renderConnectionError(args.record as Node)) + + return h('div', { class: 'flex items-center' }, template) }, sorter: true, pure: true, diff --git a/e2e/tests/node-connection-error.spec.ts b/e2e/tests/node-connection-error.spec.ts new file mode 100644 index 00000000..5957d999 --- /dev/null +++ b/e2e/tests/node-connection-error.spec.ts @@ -0,0 +1,47 @@ +import { expect, test } from '@playwright/test' +import { expectTableRows, gotoRoute } from './helpers' + +const connectionError = + 'node HTTP probe failed: tls: failed to verify certificate: x509: certificate has expired or is not yet valid: ' + + 'current time 2026-08-13T00:16:06Z is before 2026-08-13T00:26:10Z' + +test('node list exposes the latest connection error', async ({ page }, testInfo) => { + await page.route('**/api/nodes?**', async route => { + if (route.request().method() !== 'GET') { + await route.continue() + return + } + + const response = await route.fetch() + const payload = await response.json() + const node = payload.data?.find((item: { name?: string }) => item.name === 'demo-node-2') + expect(node, 'The demo node fixture was not returned by /api/nodes').toBeTruthy() + Object.assign(node, { + status: false, + connection_error: connectionError, + connection_error_code: 'clock_skew', + connection_error_at: '2026-08-13T00:16:06Z', + }) + + await route.fulfill({ response, json: payload }) + }) + + await gotoRoute(page, '/nodes') + const rows = await expectTableRows(page, 2) + const nodeRow = rows.filter({ hasText: 'demo-node-2' }).first() + await expect(nodeRow).toContainText('Offline') + + const errorIndicator = nodeRow.getByRole('button', { name: 'System clocks are out of sync' }) + await expect(errorIndicator).toBeVisible() + await errorIndicator.hover() + const popover = page.locator('.ant-popover').filter({ hasText: 'System clocks are out of sync' }) + await expect(popover).toBeVisible() + await expect(popover).toContainText('Synchronize the host system time on both the controller and the node.') + await expect(popover).toContainText('sudo timedatectl set-ntp true') + await expect(popover).toContainText('The node will reconnect automatically after the clocks are synchronized.') + + await page.screenshot({ + path: testInfo.outputPath('node-connection-error.png'), + fullPage: true, + }) +}) diff --git a/internal/analytic/node.go b/internal/analytic/node.go index 23e3bc6d..d66a71d4 100644 --- a/internal/analytic/node.go +++ b/internal/analytic/node.go @@ -38,10 +38,17 @@ type NodeStat struct { UpstreamStatusMap map[string]*upstream.Status `json:"upstream_status_map"` } +type NodeConnectionErrorCode string + +const NodeConnectionErrorClockSkew NodeConnectionErrorCode = "clock_skew" + type Node struct { *model.Node NodeStat NodeInfo + ConnectionError string `json:"connection_error,omitempty"` + ConnectionErrorCode NodeConnectionErrorCode `json:"connection_error_code,omitempty"` + ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"` } var nodeMapMu sync.RWMutex diff --git a/internal/analytic/node_record.go b/internal/analytic/node_record.go index a01181a2..090ebb8b 100644 --- a/internal/analytic/node_record.go +++ b/internal/analytic/node_record.go @@ -3,8 +3,12 @@ package analytic import ( "bytes" "context" + "crypto/x509" "encoding/json" + "errors" + "fmt" "net/http" + "strings" "sync" "time" @@ -97,7 +101,7 @@ func shouldRetry(nodeID uint64) bool { return !now.Before(state.NextRetry) } -func markConnectionFailure(nodeID uint64) int { +func markConnectionFailure(nodeID uint64, connectionErr error) int { retryMutex.Lock() state, exists := retryStates[nodeID] if !exists { @@ -109,6 +113,15 @@ func markConnectionFailure(nodeID uint64) int { state.NextRetry = time.Now().Add(calculateNextRetryInterval(state.FailureCount)) retryMutex.Unlock() + nodeMapMu.Lock() + if node := NodeMap[nodeID]; node != nil { + failedAt := time.Now() + node.ConnectionError = connectionErr.Error() + node.ConnectionErrorCode = classifyNodeConnectionError(connectionErr, failedAt) + node.ConnectionErrorAt = &failedAt + } + nodeMapMu.Unlock() + markNodeOfflineIfStale(nodeID, nodeOfflineTimeout) return failureCount } @@ -124,9 +137,43 @@ func markConnectionSuccess(nodeID uint64) bool { state.FailureCount = 0 state.NextRetry = time.Now() retryMutex.Unlock() + + nodeMapMu.Lock() + if node := NodeMap[nodeID]; node != nil { + node.ConnectionError = "" + node.ConnectionErrorCode = "" + node.ConnectionErrorAt = nil + } + nodeMapMu.Unlock() return recovered } +func classifyNodeConnectionError(connectionErr error, now time.Time) NodeConnectionErrorCode { + if connectionErr == nil { + return "" + } + + var certificateInvalidError x509.CertificateInvalidError + if errors.As(connectionErr, &certificateInvalidError) && + certificateInvalidError.Cert != nil && + now.Before(certificateInvalidError.Cert.NotBefore) { + return NodeConnectionErrorClockSkew + } + + message := connectionErr.Error() + if strings.Contains(message, "current time") && + strings.Contains(message, "is before") && + strings.Contains(message, "not yet valid") { + return NodeConnectionErrorClockSkew + } + if strings.Contains(message, "node signature creation time is in the future") || + strings.Contains(message, "node signature is expired") { + return NodeConnectionErrorClockSkew + } + + return "" +} + // ReloadNodesStatus asks the single monitor loop started by the kernel to // rebuild its workers. It deliberately does not start another monitor: doing // so lets two connections race to publish status for the same node. @@ -296,7 +343,7 @@ func runNodeStatusWorker(ctx context.Context, node *model.Node) { if ctx.Err() != nil { return } - failureCount := markConnectionFailure(node.ID) + failureCount := markConnectionFailure(node.ID, err) if failureCount == 1 { logger.Warnf("Node status connection failed for node %d (%q): %v", node.ID, node.Name, err) } @@ -363,7 +410,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error { NodeMap[nodeModel.ID].Node = nodeModel } nodeMapMu.Unlock() - return err + return fmt.Errorf("node HTTP probe failed: %w", err) } nodeMapMu.Lock() @@ -377,12 +424,12 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error { u, err := nodeModel.GetWebSocketURL("/api/analytic/intro") if err != nil { - return err + return fmt.Errorf("build node WebSocket URL: %w", err) } header := http.Header{} if err := nodeauth.SignWebSocketHeaders(nodeModel, u, header); err != nil { - return err + return fmt.Errorf("sign node WebSocket request: %w", err) } dial := &websocket.Dialer{ @@ -392,7 +439,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error { c, _, err := dial.DialContext(scopeCtx, u, header) if err != nil { - return err + return fmt.Errorf("connect node WebSocket: %w", err) } defer func() { @@ -448,7 +495,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error { var rawMsg json.RawMessage err = c.ReadJSON(&rawMsg) if err != nil { - return err + return fmt.Errorf("read node WebSocket status: %w", err) } nodeMapMu.Lock() diff --git a/internal/analytic/node_record_test.go b/internal/analytic/node_record_test.go index d29a97e2..2708ad88 100644 --- a/internal/analytic/node_record_test.go +++ b/internal/analytic/node_record_test.go @@ -2,6 +2,7 @@ package analytic import ( "context" + "crypto/x509" "encoding/json" "net/http" "net/http/httptest" @@ -123,7 +124,7 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) { retryMutex.Unlock() }) - markConnectionFailure(nodeID) + markConnectionFailure(nodeID, context.DeadlineExceeded) nodeMapMu.RLock() node := cloneNode(NodeMap[nodeID]) @@ -134,6 +135,9 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) { if !node.ResponseAt.Equal(lastResponse) { t.Fatalf("expected last successful response time to be preserved, got %v", node.ResponseAt) } + if node.ConnectionError != context.DeadlineExceeded.Error() || node.ConnectionErrorAt == nil { + t.Fatalf("expected the latest connection error to be retained, got %q at %v", node.ConnectionError, node.ConnectionErrorAt) + } } func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) { @@ -155,7 +159,7 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) { retryMutex.Unlock() }) - markConnectionFailure(nodeID) + markConnectionFailure(nodeID, context.DeadlineExceeded) nodeMapMu.RLock() node := cloneNode(NodeMap[nodeID]) @@ -168,6 +172,40 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) { } } +func TestConnectionFailureClassifiesClockSkew(t *testing.T) { + nodeID := uint64(50) + now := time.Now() + certificate := &x509.Certificate{NotBefore: now.Add(10 * time.Minute)} + + nodeMapMu.Lock() + NodeMap[nodeID] = &Node{} + nodeMapMu.Unlock() + retryMutex.Lock() + delete(retryStates, nodeID) + retryMutex.Unlock() + t.Cleanup(func() { + nodeMapMu.Lock() + delete(NodeMap, nodeID) + nodeMapMu.Unlock() + retryMutex.Lock() + delete(retryStates, nodeID) + retryMutex.Unlock() + }) + + markConnectionFailure(nodeID, x509.CertificateInvalidError{ + Cert: certificate, + Reason: x509.Expired, + Detail: "current time is before the certificate validity period", + }) + + nodeMapMu.RLock() + node := cloneNode(NodeMap[nodeID]) + nodeMapMu.RUnlock() + if node.ConnectionErrorCode != NodeConnectionErrorClockSkew { + t.Fatalf("connection error code = %q, want %q", node.ConnectionErrorCode, NodeConnectionErrorClockSkew) + } +} + func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) { nodeID := uint64(45) retryMutex.Lock() @@ -180,8 +218,20 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) { retryMutex.Lock() delete(retryStates, nodeID) retryMutex.Unlock() + nodeMapMu.Lock() + delete(NodeMap, nodeID) + nodeMapMu.Unlock() }) + failedAt := time.Now() + nodeMapMu.Lock() + NodeMap[nodeID] = &Node{ + ConnectionError: context.DeadlineExceeded.Error(), + ConnectionErrorCode: NodeConnectionErrorClockSkew, + ConnectionErrorAt: &failedAt, + } + nodeMapMu.Unlock() + if !markConnectionSuccess(nodeID) { t.Fatal("expected a successful sample after failures to report recovery") } @@ -195,6 +245,13 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) { if state.NextRetry.After(time.Now()) { t.Fatalf("expected retry to be immediately available, got %v", state.NextRetry) } + nodeMapMu.RLock() + node := cloneNode(NodeMap[nodeID]) + nodeMapMu.RUnlock() + if node.ConnectionError != "" || node.ConnectionErrorCode != "" || node.ConnectionErrorAt != nil { + t.Fatalf("expected a successful sample to clear the connection error, got %q (%q) at %v", + node.ConnectionError, node.ConnectionErrorCode, node.ConnectionErrorAt) + } } func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) { @@ -208,10 +265,10 @@ func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) { retryMutex.Unlock() }) - if count := markConnectionFailure(nodeID); count != 1 { + if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 1 { t.Fatalf("first failure count = %d, want 1", count) } - if count := markConnectionFailure(nodeID); count != 2 { + if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 2 { t.Fatalf("second failure count = %d, want 2", count) } } diff --git a/internal/analytic/node_time_skew_test.go b/internal/analytic/node_time_skew_test.go new file mode 100644 index 00000000..dc0616f9 --- /dev/null +++ b/internal/analytic/node_time_skew_test.go @@ -0,0 +1,129 @@ +package analytic + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/0xJacky/Nginx-UI/internal/nodeauth" + "github.com/0xJacky/Nginx-UI/model" + "github.com/stretchr/testify/require" +) + +const fourSecondNodeClockSkew = 4 * time.Second + +func TestNodeTLSHandshakeFailsWhileCertificateIsFourSecondsInFuture(t *testing.T) { + const secret = "time-skew-node-secret" + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("X-Node-Secret") != secret { + writer.WriteHeader(http.StatusForbidden) + return + } + writer.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(writer).Encode(NodeInfo{Version: "time-skew-test"}); err != nil { + t.Errorf("encode node info: %v", err) + } + })) + + serverIP := server.Listener.Addr().(*net.TCPAddr).IP + serverCertificate, rootCertificate, notBefore := newFutureNodeCertificate(t, serverIP, fourSecondNodeClockSkew) + server.TLS = &tls.Config{ + Certificates: []tls.Certificate{serverCertificate}, + MinVersion: tls.VersionTLS12, + } + server.StartTLS() + t.Cleanup(server.Close) + + node := &model.Node{ + Model: model.Model{ID: 49}, + Name: "four-second-clock-skew", + URL: server.URL, + } + setupLegacyNodeAuthForTest(t, node, secret) + + roots := x509.NewCertPool() + roots.AddCert(rootCertificate) + client := &http.Client{ + Transport: nodeauth.NewTransport(node, &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: roots, + MinVersion: tls.VersionTLS12, + }}), + Timeout: 2 * time.Second, + } + t.Cleanup(client.CloseIdleConnections) + + requestURL := server.URL + "/api/node" + response, err := client.Get(requestURL) + require.Nil(t, response) + require.ErrorContains(t, err, "is not yet valid") + + wait := time.Until(notBefore.Add(100 * time.Millisecond)) + require.Positive(t, wait, "test setup took longer than the four-second skew window") + timer := time.NewTimer(wait) + t.Cleanup(func() { timer.Stop() }) + <-timer.C + + response, err = client.Get(requestURL) + require.NoError(t, err) + t.Cleanup(func() { response.Body.Close() }) + require.Equal(t, http.StatusOK, response.StatusCode) +} + +func newFutureNodeCertificate(t *testing.T, serverIP net.IP, futureOffset time.Duration) ( + tls.Certificate, + *x509.Certificate, + time.Time, +) { + t.Helper() + + rootPublicKey, rootPrivateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + rootTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Nginx UI node time-skew test root"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPublicKey, rootPrivateKey) + require.NoError(t, err) + rootCertificate, err := x509.ParseCertificate(rootDER) + require.NoError(t, err) + + leafPublicKey, leafPrivateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + notBefore := time.Now().Add(futureOffset).Truncate(time.Second) + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "Nginx UI child node"}, + IPAddresses: []net.IP{serverIP}, + NotBefore: notBefore, + NotAfter: notBefore.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, rootTemplate, leafPublicKey, rootPrivateKey) + require.NoError(t, err) + + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(leafPrivateKey) + require.NoError(t, err) + privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER}) + serverCertificate, err := tls.X509KeyPair(certificatePEM, privateKeyPEM) + require.NoError(t, err) + + return serverCertificate, rootCertificate, notBefore +}