feat(cluster): explain why a node is unreachable

This commit is contained in:
0xJacky
2026-08-13 10:19:04 +08:00
parent 8deb8578f7
commit acd32b85ee
9 changed files with 396 additions and 23 deletions

View File

@@ -31,16 +31,19 @@ type nodeMutationRequest struct {
}
type nodeResponse struct {
ID uint64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
URL string `json:"url"`
Enabled bool `json:"enabled"`
AuthMethod string `json:"auth_method"`
HasCredential bool `json:"has_credential"`
CredentialStatus string `json:"credential_status"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
ID uint64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
URL string `json:"url"`
Enabled bool `json:"enabled"`
AuthMethod string `json:"auth_method"`
HasCredential bool `json:"has_credential"`
CredentialStatus string `json:"credential_status"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
ConnectionError string `json:"connection_error,omitempty"`
ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
// LegacySecret only ever carries the redaction sentinel, which tells the
// edit form a secret is stored without putting it in a list response.
LegacySecret string `json:"legacy_secret,omitempty"`
@@ -68,6 +71,9 @@ func newNodeResponse(node *model.Node) nodeResponse {
if analyticNode != nil {
response.NodeStat = analyticNode.NodeStat
response.NodeInfo = analyticNode.NodeInfo
response.ConnectionError = analyticNode.ConnectionError
response.ConnectionErrorCode = analyticNode.ConnectionErrorCode
response.ConnectionErrorAt = analyticNode.ConnectionErrorAt
}
return response
}

View File

@@ -14,6 +14,9 @@ export interface Node extends ModelBase {
credential_status: 'active' | 'unpaired' | 'rotating' | 'revoked'
last_credential_use_at?: string
response_at?: string
connection_error?: string
connection_error_code?: 'clock_skew'
connection_error_at?: string
}
export interface NodeStatus {

View File

@@ -103,6 +103,9 @@ export const useNodeAvailabilityStore = defineStore('nodeAvailability', () => {
auth_method: node.auth_method,
has_credential: node.has_credential,
credential_status: node.credential_status,
connection_error: node.connection_error,
connection_error_code: node.connection_error_code,
connection_error_at: node.connection_error_at,
enabled: true,
}
})

View File

@@ -1,8 +1,9 @@
import type { CustomRenderArgs, StdTableColumn } from '@uozi-admin/curd'
import type { JSX } from 'vue/jsx-runtime'
import type { Node } from '@/api/node'
import { ExclamationCircleOutlined, InfoCircleOutlined } from '@ant-design/icons-vue'
import { datetimeRender } from '@uozi-admin/curd'
import { Badge, InputPassword, Tag } from 'ant-design-vue'
import { Badge, InputPassword, Popover, Tag } from 'ant-design-vue'
import { h } from 'vue'
import nodeApi from '@/api/node'
import { SensitiveInput } from '@/components/SensitiveString'
@@ -15,6 +16,76 @@ const unhealthyCredentialMap: Record<string, { color: string, text: () => string
revoked: { color: 'red', text: () => $gettext('Revoked') },
}
function renderConnectionErrorContent(record: Node) {
if (record.connection_error_code !== 'clock_skew') {
return (
<div class="max-w-[400px] text-sm leading-6">
<p class="m-0">
{$gettext('Check the node URL, network, TLS certificate, and authentication settings.')}
</p>
<details class="mt-3">
<summary class="cursor-pointer select-none font-medium">
{$gettext('Technical details')}
</summary>
<pre class="mb-0 mt-2 max-h-48 overflow-auto whitespace-pre-wrap break-all rounded bg-gray-100 p-3 text-xs dark:bg-gray-800">
{record.connection_error}
</pre>
</details>
</div>
)
}
return (
<div class="w-[400px] max-w-[calc(100vw-48px)] text-sm leading-6">
<p class="m-0">
{$gettext('The controller time is earlier than the node certificate validity start time. TLS stops the connection before node authentication.')}
</p>
<ol class="my-3 pl-5">
<li>{$gettext('Synchronize the host system time on both the controller and the node.')}</li>
<li>{$gettext('If Nginx UI runs in Docker, correct the Docker host clock instead of the container clock.')}</li>
</ol>
<p class="mb-1 mt-0 font-medium">
{$gettext('On Linux hosts, check and enable network time synchronization:')}
</p>
<pre class="my-0 overflow-auto rounded bg-gray-100 px-3 py-2 text-xs dark:bg-gray-800">
{'timedatectl status\nsudo timedatectl set-ntp true'}
</pre>
<p class="mb-0 mt-3 text-gray-600 dark:text-gray-300">
{$gettext('The node will reconnect automatically after the clocks are synchronized.')}
</p>
<details class="mt-3">
<summary class="cursor-pointer select-none font-medium">
{$gettext('Technical details')}
</summary>
<pre class="mb-0 mt-2 max-h-48 overflow-auto whitespace-pre-wrap break-all rounded bg-gray-100 p-3 text-xs dark:bg-gray-800">
{record.connection_error}
</pre>
</details>
</div>
)
}
function renderConnectionError(record: Node) {
const isClockSkew = record.connection_error_code === 'clock_skew'
const title = isClockSkew
? $gettext('System clocks are out of sync')
: $gettext('Node connection failed')
return h(Popover, {
content: renderConnectionErrorContent(record),
placement: 'rightTop',
title: h('div', { class: 'flex items-center gap-2' }, [
h(ExclamationCircleOutlined, { class: isClockSkew ? 'text-orange-500' : 'text-red-500' }),
h('span', title),
]),
trigger: ['hover', 'focus', 'click'],
}, () => h('button', {
'type': 'button',
'aria-label': title,
'class': 'ml-1 inline-flex cursor-help items-center border-0 bg-transparent p-0 text-red-500',
}, h(InfoCircleOutlined)))
}
const columns: StdTableColumn[] = [{
title: () => $gettext('Name'),
dataIndex: 'name',
@@ -104,7 +175,10 @@ const columns: StdTableColumn[] = [{
template.push(<span>{$gettext('Disabled')}</span>)
}
return h('div', template)
if (args.record.connection_error)
template.push(renderConnectionError(args.record as Node))
return h('div', { class: 'flex items-center' }, template)
},
sorter: true,
pure: true,

View File

@@ -0,0 +1,47 @@
import { expect, test } from '@playwright/test'
import { expectTableRows, gotoRoute } from './helpers'
const connectionError =
'node HTTP probe failed: tls: failed to verify certificate: x509: certificate has expired or is not yet valid: ' +
'current time 2026-08-13T00:16:06Z is before 2026-08-13T00:26:10Z'
test('node list exposes the latest connection error', async ({ page }, testInfo) => {
await page.route('**/api/nodes?**', async route => {
if (route.request().method() !== 'GET') {
await route.continue()
return
}
const response = await route.fetch()
const payload = await response.json()
const node = payload.data?.find((item: { name?: string }) => item.name === 'demo-node-2')
expect(node, 'The demo node fixture was not returned by /api/nodes').toBeTruthy()
Object.assign(node, {
status: false,
connection_error: connectionError,
connection_error_code: 'clock_skew',
connection_error_at: '2026-08-13T00:16:06Z',
})
await route.fulfill({ response, json: payload })
})
await gotoRoute(page, '/nodes')
const rows = await expectTableRows(page, 2)
const nodeRow = rows.filter({ hasText: 'demo-node-2' }).first()
await expect(nodeRow).toContainText('Offline')
const errorIndicator = nodeRow.getByRole('button', { name: 'System clocks are out of sync' })
await expect(errorIndicator).toBeVisible()
await errorIndicator.hover()
const popover = page.locator('.ant-popover').filter({ hasText: 'System clocks are out of sync' })
await expect(popover).toBeVisible()
await expect(popover).toContainText('Synchronize the host system time on both the controller and the node.')
await expect(popover).toContainText('sudo timedatectl set-ntp true')
await expect(popover).toContainText('The node will reconnect automatically after the clocks are synchronized.')
await page.screenshot({
path: testInfo.outputPath('node-connection-error.png'),
fullPage: true,
})
})

View File

@@ -38,10 +38,17 @@ type NodeStat struct {
UpstreamStatusMap map[string]*upstream.Status `json:"upstream_status_map"`
}
type NodeConnectionErrorCode string
const NodeConnectionErrorClockSkew NodeConnectionErrorCode = "clock_skew"
type Node struct {
*model.Node
NodeStat
NodeInfo
ConnectionError string `json:"connection_error,omitempty"`
ConnectionErrorCode NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
}
var nodeMapMu sync.RWMutex

View File

@@ -3,8 +3,12 @@ package analytic
import (
"bytes"
"context"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"time"
@@ -97,7 +101,7 @@ func shouldRetry(nodeID uint64) bool {
return !now.Before(state.NextRetry)
}
func markConnectionFailure(nodeID uint64) int {
func markConnectionFailure(nodeID uint64, connectionErr error) int {
retryMutex.Lock()
state, exists := retryStates[nodeID]
if !exists {
@@ -109,6 +113,15 @@ func markConnectionFailure(nodeID uint64) int {
state.NextRetry = time.Now().Add(calculateNextRetryInterval(state.FailureCount))
retryMutex.Unlock()
nodeMapMu.Lock()
if node := NodeMap[nodeID]; node != nil {
failedAt := time.Now()
node.ConnectionError = connectionErr.Error()
node.ConnectionErrorCode = classifyNodeConnectionError(connectionErr, failedAt)
node.ConnectionErrorAt = &failedAt
}
nodeMapMu.Unlock()
markNodeOfflineIfStale(nodeID, nodeOfflineTimeout)
return failureCount
}
@@ -124,9 +137,43 @@ func markConnectionSuccess(nodeID uint64) bool {
state.FailureCount = 0
state.NextRetry = time.Now()
retryMutex.Unlock()
nodeMapMu.Lock()
if node := NodeMap[nodeID]; node != nil {
node.ConnectionError = ""
node.ConnectionErrorCode = ""
node.ConnectionErrorAt = nil
}
nodeMapMu.Unlock()
return recovered
}
func classifyNodeConnectionError(connectionErr error, now time.Time) NodeConnectionErrorCode {
if connectionErr == nil {
return ""
}
var certificateInvalidError x509.CertificateInvalidError
if errors.As(connectionErr, &certificateInvalidError) &&
certificateInvalidError.Cert != nil &&
now.Before(certificateInvalidError.Cert.NotBefore) {
return NodeConnectionErrorClockSkew
}
message := connectionErr.Error()
if strings.Contains(message, "current time") &&
strings.Contains(message, "is before") &&
strings.Contains(message, "not yet valid") {
return NodeConnectionErrorClockSkew
}
if strings.Contains(message, "node signature creation time is in the future") ||
strings.Contains(message, "node signature is expired") {
return NodeConnectionErrorClockSkew
}
return ""
}
// ReloadNodesStatus asks the single monitor loop started by the kernel to
// rebuild its workers. It deliberately does not start another monitor: doing
// so lets two connections race to publish status for the same node.
@@ -296,7 +343,7 @@ func runNodeStatusWorker(ctx context.Context, node *model.Node) {
if ctx.Err() != nil {
return
}
failureCount := markConnectionFailure(node.ID)
failureCount := markConnectionFailure(node.ID, err)
if failureCount == 1 {
logger.Warnf("Node status connection failed for node %d (%q): %v", node.ID, node.Name, err)
}
@@ -363,7 +410,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
NodeMap[nodeModel.ID].Node = nodeModel
}
nodeMapMu.Unlock()
return err
return fmt.Errorf("node HTTP probe failed: %w", err)
}
nodeMapMu.Lock()
@@ -377,12 +424,12 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
u, err := nodeModel.GetWebSocketURL("/api/analytic/intro")
if err != nil {
return err
return fmt.Errorf("build node WebSocket URL: %w", err)
}
header := http.Header{}
if err := nodeauth.SignWebSocketHeaders(nodeModel, u, header); err != nil {
return err
return fmt.Errorf("sign node WebSocket request: %w", err)
}
dial := &websocket.Dialer{
@@ -392,7 +439,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
c, _, err := dial.DialContext(scopeCtx, u, header)
if err != nil {
return err
return fmt.Errorf("connect node WebSocket: %w", err)
}
defer func() {
@@ -448,7 +495,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
var rawMsg json.RawMessage
err = c.ReadJSON(&rawMsg)
if err != nil {
return err
return fmt.Errorf("read node WebSocket status: %w", err)
}
nodeMapMu.Lock()

View File

@@ -2,6 +2,7 @@ package analytic
import (
"context"
"crypto/x509"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -123,7 +124,7 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) {
retryMutex.Unlock()
})
markConnectionFailure(nodeID)
markConnectionFailure(nodeID, context.DeadlineExceeded)
nodeMapMu.RLock()
node := cloneNode(NodeMap[nodeID])
@@ -134,6 +135,9 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) {
if !node.ResponseAt.Equal(lastResponse) {
t.Fatalf("expected last successful response time to be preserved, got %v", node.ResponseAt)
}
if node.ConnectionError != context.DeadlineExceeded.Error() || node.ConnectionErrorAt == nil {
t.Fatalf("expected the latest connection error to be retained, got %q at %v", node.ConnectionError, node.ConnectionErrorAt)
}
}
func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
@@ -155,7 +159,7 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
retryMutex.Unlock()
})
markConnectionFailure(nodeID)
markConnectionFailure(nodeID, context.DeadlineExceeded)
nodeMapMu.RLock()
node := cloneNode(NodeMap[nodeID])
@@ -168,6 +172,40 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
}
}
func TestConnectionFailureClassifiesClockSkew(t *testing.T) {
nodeID := uint64(50)
now := time.Now()
certificate := &x509.Certificate{NotBefore: now.Add(10 * time.Minute)}
nodeMapMu.Lock()
NodeMap[nodeID] = &Node{}
nodeMapMu.Unlock()
retryMutex.Lock()
delete(retryStates, nodeID)
retryMutex.Unlock()
t.Cleanup(func() {
nodeMapMu.Lock()
delete(NodeMap, nodeID)
nodeMapMu.Unlock()
retryMutex.Lock()
delete(retryStates, nodeID)
retryMutex.Unlock()
})
markConnectionFailure(nodeID, x509.CertificateInvalidError{
Cert: certificate,
Reason: x509.Expired,
Detail: "current time is before the certificate validity period",
})
nodeMapMu.RLock()
node := cloneNode(NodeMap[nodeID])
nodeMapMu.RUnlock()
if node.ConnectionErrorCode != NodeConnectionErrorClockSkew {
t.Fatalf("connection error code = %q, want %q", node.ConnectionErrorCode, NodeConnectionErrorClockSkew)
}
}
func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
nodeID := uint64(45)
retryMutex.Lock()
@@ -180,8 +218,20 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
retryMutex.Lock()
delete(retryStates, nodeID)
retryMutex.Unlock()
nodeMapMu.Lock()
delete(NodeMap, nodeID)
nodeMapMu.Unlock()
})
failedAt := time.Now()
nodeMapMu.Lock()
NodeMap[nodeID] = &Node{
ConnectionError: context.DeadlineExceeded.Error(),
ConnectionErrorCode: NodeConnectionErrorClockSkew,
ConnectionErrorAt: &failedAt,
}
nodeMapMu.Unlock()
if !markConnectionSuccess(nodeID) {
t.Fatal("expected a successful sample after failures to report recovery")
}
@@ -195,6 +245,13 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
if state.NextRetry.After(time.Now()) {
t.Fatalf("expected retry to be immediately available, got %v", state.NextRetry)
}
nodeMapMu.RLock()
node := cloneNode(NodeMap[nodeID])
nodeMapMu.RUnlock()
if node.ConnectionError != "" || node.ConnectionErrorCode != "" || node.ConnectionErrorAt != nil {
t.Fatalf("expected a successful sample to clear the connection error, got %q (%q) at %v",
node.ConnectionError, node.ConnectionErrorCode, node.ConnectionErrorAt)
}
}
func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) {
@@ -208,10 +265,10 @@ func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) {
retryMutex.Unlock()
})
if count := markConnectionFailure(nodeID); count != 1 {
if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 1 {
t.Fatalf("first failure count = %d, want 1", count)
}
if count := markConnectionFailure(nodeID); count != 2 {
if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 2 {
t.Fatalf("second failure count = %d, want 2", count)
}
}

View File

@@ -0,0 +1,129 @@
package analytic
import (
"crypto/ed25519"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"math/big"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/0xJacky/Nginx-UI/internal/nodeauth"
"github.com/0xJacky/Nginx-UI/model"
"github.com/stretchr/testify/require"
)
const fourSecondNodeClockSkew = 4 * time.Second
func TestNodeTLSHandshakeFailsWhileCertificateIsFourSecondsInFuture(t *testing.T) {
const secret = "time-skew-node-secret"
server := httptest.NewUnstartedServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Header.Get("X-Node-Secret") != secret {
writer.WriteHeader(http.StatusForbidden)
return
}
writer.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(writer).Encode(NodeInfo{Version: "time-skew-test"}); err != nil {
t.Errorf("encode node info: %v", err)
}
}))
serverIP := server.Listener.Addr().(*net.TCPAddr).IP
serverCertificate, rootCertificate, notBefore := newFutureNodeCertificate(t, serverIP, fourSecondNodeClockSkew)
server.TLS = &tls.Config{
Certificates: []tls.Certificate{serverCertificate},
MinVersion: tls.VersionTLS12,
}
server.StartTLS()
t.Cleanup(server.Close)
node := &model.Node{
Model: model.Model{ID: 49},
Name: "four-second-clock-skew",
URL: server.URL,
}
setupLegacyNodeAuthForTest(t, node, secret)
roots := x509.NewCertPool()
roots.AddCert(rootCertificate)
client := &http.Client{
Transport: nodeauth.NewTransport(node, &http.Transport{TLSClientConfig: &tls.Config{
RootCAs: roots,
MinVersion: tls.VersionTLS12,
}}),
Timeout: 2 * time.Second,
}
t.Cleanup(client.CloseIdleConnections)
requestURL := server.URL + "/api/node"
response, err := client.Get(requestURL)
require.Nil(t, response)
require.ErrorContains(t, err, "is not yet valid")
wait := time.Until(notBefore.Add(100 * time.Millisecond))
require.Positive(t, wait, "test setup took longer than the four-second skew window")
timer := time.NewTimer(wait)
t.Cleanup(func() { timer.Stop() })
<-timer.C
response, err = client.Get(requestURL)
require.NoError(t, err)
t.Cleanup(func() { response.Body.Close() })
require.Equal(t, http.StatusOK, response.StatusCode)
}
func newFutureNodeCertificate(t *testing.T, serverIP net.IP, futureOffset time.Duration) (
tls.Certificate,
*x509.Certificate,
time.Time,
) {
t.Helper()
rootPublicKey, rootPrivateKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
rootTemplate := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "Nginx UI node time-skew test root"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPublicKey, rootPrivateKey)
require.NoError(t, err)
rootCertificate, err := x509.ParseCertificate(rootDER)
require.NoError(t, err)
leafPublicKey, leafPrivateKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
notBefore := time.Now().Add(futureOffset).Truncate(time.Second)
leafTemplate := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: "Nginx UI child node"},
IPAddresses: []net.IP{serverIP},
NotBefore: notBefore,
NotAfter: notBefore.Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, rootTemplate, leafPublicKey, rootPrivateKey)
require.NoError(t, err)
certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER})
privateKeyDER, err := x509.MarshalPKCS8PrivateKey(leafPrivateKey)
require.NoError(t, err)
privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER})
serverCertificate, err := tls.X509KeyPair(certificatePEM, privateKeyPEM)
require.NoError(t, err)
return serverCertificate, rootCertificate, notBefore
}