mirror of
https://github.com/0xJacky/nginx-ui.git
synced 2026-09-03 07:24:52 +08:00
feat(cluster): explain why a node is unreachable
This commit is contained in:
@@ -31,16 +31,19 @@ type nodeMutationRequest struct {
|
||||
}
|
||||
|
||||
type nodeResponse struct {
|
||||
ID uint64 `json:"id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Enabled bool `json:"enabled"`
|
||||
AuthMethod string `json:"auth_method"`
|
||||
HasCredential bool `json:"has_credential"`
|
||||
CredentialStatus string `json:"credential_status"`
|
||||
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
|
||||
ID uint64 `json:"id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Enabled bool `json:"enabled"`
|
||||
AuthMethod string `json:"auth_method"`
|
||||
HasCredential bool `json:"has_credential"`
|
||||
CredentialStatus string `json:"credential_status"`
|
||||
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
|
||||
ConnectionError string `json:"connection_error,omitempty"`
|
||||
ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
|
||||
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
|
||||
// LegacySecret only ever carries the redaction sentinel, which tells the
|
||||
// edit form a secret is stored without putting it in a list response.
|
||||
LegacySecret string `json:"legacy_secret,omitempty"`
|
||||
@@ -68,6 +71,9 @@ func newNodeResponse(node *model.Node) nodeResponse {
|
||||
if analyticNode != nil {
|
||||
response.NodeStat = analyticNode.NodeStat
|
||||
response.NodeInfo = analyticNode.NodeInfo
|
||||
response.ConnectionError = analyticNode.ConnectionError
|
||||
response.ConnectionErrorCode = analyticNode.ConnectionErrorCode
|
||||
response.ConnectionErrorAt = analyticNode.ConnectionErrorAt
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
@@ -14,6 +14,9 @@ export interface Node extends ModelBase {
|
||||
credential_status: 'active' | 'unpaired' | 'rotating' | 'revoked'
|
||||
last_credential_use_at?: string
|
||||
response_at?: string
|
||||
connection_error?: string
|
||||
connection_error_code?: 'clock_skew'
|
||||
connection_error_at?: string
|
||||
}
|
||||
|
||||
export interface NodeStatus {
|
||||
|
||||
@@ -103,6 +103,9 @@ export const useNodeAvailabilityStore = defineStore('nodeAvailability', () => {
|
||||
auth_method: node.auth_method,
|
||||
has_credential: node.has_credential,
|
||||
credential_status: node.credential_status,
|
||||
connection_error: node.connection_error,
|
||||
connection_error_code: node.connection_error_code,
|
||||
connection_error_at: node.connection_error_at,
|
||||
enabled: true,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import type { CustomRenderArgs, StdTableColumn } from '@uozi-admin/curd'
|
||||
import type { JSX } from 'vue/jsx-runtime'
|
||||
import type { Node } from '@/api/node'
|
||||
import { ExclamationCircleOutlined, InfoCircleOutlined } from '@ant-design/icons-vue'
|
||||
import { datetimeRender } from '@uozi-admin/curd'
|
||||
import { Badge, InputPassword, Tag } from 'ant-design-vue'
|
||||
import { Badge, InputPassword, Popover, Tag } from 'ant-design-vue'
|
||||
import { h } from 'vue'
|
||||
import nodeApi from '@/api/node'
|
||||
import { SensitiveInput } from '@/components/SensitiveString'
|
||||
@@ -15,6 +16,76 @@ const unhealthyCredentialMap: Record<string, { color: string, text: () => string
|
||||
revoked: { color: 'red', text: () => $gettext('Revoked') },
|
||||
}
|
||||
|
||||
function renderConnectionErrorContent(record: Node) {
|
||||
if (record.connection_error_code !== 'clock_skew') {
|
||||
return (
|
||||
<div class="max-w-[400px] text-sm leading-6">
|
||||
<p class="m-0">
|
||||
{$gettext('Check the node URL, network, TLS certificate, and authentication settings.')}
|
||||
</p>
|
||||
<details class="mt-3">
|
||||
<summary class="cursor-pointer select-none font-medium">
|
||||
{$gettext('Technical details')}
|
||||
</summary>
|
||||
<pre class="mb-0 mt-2 max-h-48 overflow-auto whitespace-pre-wrap break-all rounded bg-gray-100 p-3 text-xs dark:bg-gray-800">
|
||||
{record.connection_error}
|
||||
</pre>
|
||||
</details>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div class="w-[400px] max-w-[calc(100vw-48px)] text-sm leading-6">
|
||||
<p class="m-0">
|
||||
{$gettext('The controller time is earlier than the node certificate validity start time. TLS stops the connection before node authentication.')}
|
||||
</p>
|
||||
<ol class="my-3 pl-5">
|
||||
<li>{$gettext('Synchronize the host system time on both the controller and the node.')}</li>
|
||||
<li>{$gettext('If Nginx UI runs in Docker, correct the Docker host clock instead of the container clock.')}</li>
|
||||
</ol>
|
||||
<p class="mb-1 mt-0 font-medium">
|
||||
{$gettext('On Linux hosts, check and enable network time synchronization:')}
|
||||
</p>
|
||||
<pre class="my-0 overflow-auto rounded bg-gray-100 px-3 py-2 text-xs dark:bg-gray-800">
|
||||
{'timedatectl status\nsudo timedatectl set-ntp true'}
|
||||
</pre>
|
||||
<p class="mb-0 mt-3 text-gray-600 dark:text-gray-300">
|
||||
{$gettext('The node will reconnect automatically after the clocks are synchronized.')}
|
||||
</p>
|
||||
<details class="mt-3">
|
||||
<summary class="cursor-pointer select-none font-medium">
|
||||
{$gettext('Technical details')}
|
||||
</summary>
|
||||
<pre class="mb-0 mt-2 max-h-48 overflow-auto whitespace-pre-wrap break-all rounded bg-gray-100 p-3 text-xs dark:bg-gray-800">
|
||||
{record.connection_error}
|
||||
</pre>
|
||||
</details>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function renderConnectionError(record: Node) {
|
||||
const isClockSkew = record.connection_error_code === 'clock_skew'
|
||||
const title = isClockSkew
|
||||
? $gettext('System clocks are out of sync')
|
||||
: $gettext('Node connection failed')
|
||||
|
||||
return h(Popover, {
|
||||
content: renderConnectionErrorContent(record),
|
||||
placement: 'rightTop',
|
||||
title: h('div', { class: 'flex items-center gap-2' }, [
|
||||
h(ExclamationCircleOutlined, { class: isClockSkew ? 'text-orange-500' : 'text-red-500' }),
|
||||
h('span', title),
|
||||
]),
|
||||
trigger: ['hover', 'focus', 'click'],
|
||||
}, () => h('button', {
|
||||
'type': 'button',
|
||||
'aria-label': title,
|
||||
'class': 'ml-1 inline-flex cursor-help items-center border-0 bg-transparent p-0 text-red-500',
|
||||
}, h(InfoCircleOutlined)))
|
||||
}
|
||||
|
||||
const columns: StdTableColumn[] = [{
|
||||
title: () => $gettext('Name'),
|
||||
dataIndex: 'name',
|
||||
@@ -104,7 +175,10 @@ const columns: StdTableColumn[] = [{
|
||||
template.push(<span>{$gettext('Disabled')}</span>)
|
||||
}
|
||||
|
||||
return h('div', template)
|
||||
if (args.record.connection_error)
|
||||
template.push(renderConnectionError(args.record as Node))
|
||||
|
||||
return h('div', { class: 'flex items-center' }, template)
|
||||
},
|
||||
sorter: true,
|
||||
pure: true,
|
||||
|
||||
47
e2e/tests/node-connection-error.spec.ts
Normal file
47
e2e/tests/node-connection-error.spec.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
import { expect, test } from '@playwright/test'
|
||||
import { expectTableRows, gotoRoute } from './helpers'
|
||||
|
||||
const connectionError =
|
||||
'node HTTP probe failed: tls: failed to verify certificate: x509: certificate has expired or is not yet valid: ' +
|
||||
'current time 2026-08-13T00:16:06Z is before 2026-08-13T00:26:10Z'
|
||||
|
||||
test('node list exposes the latest connection error', async ({ page }, testInfo) => {
|
||||
await page.route('**/api/nodes?**', async route => {
|
||||
if (route.request().method() !== 'GET') {
|
||||
await route.continue()
|
||||
return
|
||||
}
|
||||
|
||||
const response = await route.fetch()
|
||||
const payload = await response.json()
|
||||
const node = payload.data?.find((item: { name?: string }) => item.name === 'demo-node-2')
|
||||
expect(node, 'The demo node fixture was not returned by /api/nodes').toBeTruthy()
|
||||
Object.assign(node, {
|
||||
status: false,
|
||||
connection_error: connectionError,
|
||||
connection_error_code: 'clock_skew',
|
||||
connection_error_at: '2026-08-13T00:16:06Z',
|
||||
})
|
||||
|
||||
await route.fulfill({ response, json: payload })
|
||||
})
|
||||
|
||||
await gotoRoute(page, '/nodes')
|
||||
const rows = await expectTableRows(page, 2)
|
||||
const nodeRow = rows.filter({ hasText: 'demo-node-2' }).first()
|
||||
await expect(nodeRow).toContainText('Offline')
|
||||
|
||||
const errorIndicator = nodeRow.getByRole('button', { name: 'System clocks are out of sync' })
|
||||
await expect(errorIndicator).toBeVisible()
|
||||
await errorIndicator.hover()
|
||||
const popover = page.locator('.ant-popover').filter({ hasText: 'System clocks are out of sync' })
|
||||
await expect(popover).toBeVisible()
|
||||
await expect(popover).toContainText('Synchronize the host system time on both the controller and the node.')
|
||||
await expect(popover).toContainText('sudo timedatectl set-ntp true')
|
||||
await expect(popover).toContainText('The node will reconnect automatically after the clocks are synchronized.')
|
||||
|
||||
await page.screenshot({
|
||||
path: testInfo.outputPath('node-connection-error.png'),
|
||||
fullPage: true,
|
||||
})
|
||||
})
|
||||
@@ -38,10 +38,17 @@ type NodeStat struct {
|
||||
UpstreamStatusMap map[string]*upstream.Status `json:"upstream_status_map"`
|
||||
}
|
||||
|
||||
type NodeConnectionErrorCode string
|
||||
|
||||
const NodeConnectionErrorClockSkew NodeConnectionErrorCode = "clock_skew"
|
||||
|
||||
type Node struct {
|
||||
*model.Node
|
||||
NodeStat
|
||||
NodeInfo
|
||||
ConnectionError string `json:"connection_error,omitempty"`
|
||||
ConnectionErrorCode NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
|
||||
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
|
||||
}
|
||||
|
||||
var nodeMapMu sync.RWMutex
|
||||
|
||||
@@ -3,8 +3,12 @@ package analytic
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -97,7 +101,7 @@ func shouldRetry(nodeID uint64) bool {
|
||||
return !now.Before(state.NextRetry)
|
||||
}
|
||||
|
||||
func markConnectionFailure(nodeID uint64) int {
|
||||
func markConnectionFailure(nodeID uint64, connectionErr error) int {
|
||||
retryMutex.Lock()
|
||||
state, exists := retryStates[nodeID]
|
||||
if !exists {
|
||||
@@ -109,6 +113,15 @@ func markConnectionFailure(nodeID uint64) int {
|
||||
state.NextRetry = time.Now().Add(calculateNextRetryInterval(state.FailureCount))
|
||||
retryMutex.Unlock()
|
||||
|
||||
nodeMapMu.Lock()
|
||||
if node := NodeMap[nodeID]; node != nil {
|
||||
failedAt := time.Now()
|
||||
node.ConnectionError = connectionErr.Error()
|
||||
node.ConnectionErrorCode = classifyNodeConnectionError(connectionErr, failedAt)
|
||||
node.ConnectionErrorAt = &failedAt
|
||||
}
|
||||
nodeMapMu.Unlock()
|
||||
|
||||
markNodeOfflineIfStale(nodeID, nodeOfflineTimeout)
|
||||
return failureCount
|
||||
}
|
||||
@@ -124,9 +137,43 @@ func markConnectionSuccess(nodeID uint64) bool {
|
||||
state.FailureCount = 0
|
||||
state.NextRetry = time.Now()
|
||||
retryMutex.Unlock()
|
||||
|
||||
nodeMapMu.Lock()
|
||||
if node := NodeMap[nodeID]; node != nil {
|
||||
node.ConnectionError = ""
|
||||
node.ConnectionErrorCode = ""
|
||||
node.ConnectionErrorAt = nil
|
||||
}
|
||||
nodeMapMu.Unlock()
|
||||
return recovered
|
||||
}
|
||||
|
||||
func classifyNodeConnectionError(connectionErr error, now time.Time) NodeConnectionErrorCode {
|
||||
if connectionErr == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
var certificateInvalidError x509.CertificateInvalidError
|
||||
if errors.As(connectionErr, &certificateInvalidError) &&
|
||||
certificateInvalidError.Cert != nil &&
|
||||
now.Before(certificateInvalidError.Cert.NotBefore) {
|
||||
return NodeConnectionErrorClockSkew
|
||||
}
|
||||
|
||||
message := connectionErr.Error()
|
||||
if strings.Contains(message, "current time") &&
|
||||
strings.Contains(message, "is before") &&
|
||||
strings.Contains(message, "not yet valid") {
|
||||
return NodeConnectionErrorClockSkew
|
||||
}
|
||||
if strings.Contains(message, "node signature creation time is in the future") ||
|
||||
strings.Contains(message, "node signature is expired") {
|
||||
return NodeConnectionErrorClockSkew
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// ReloadNodesStatus asks the single monitor loop started by the kernel to
|
||||
// rebuild its workers. It deliberately does not start another monitor: doing
|
||||
// so lets two connections race to publish status for the same node.
|
||||
@@ -296,7 +343,7 @@ func runNodeStatusWorker(ctx context.Context, node *model.Node) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
failureCount := markConnectionFailure(node.ID)
|
||||
failureCount := markConnectionFailure(node.ID, err)
|
||||
if failureCount == 1 {
|
||||
logger.Warnf("Node status connection failed for node %d (%q): %v", node.ID, node.Name, err)
|
||||
}
|
||||
@@ -363,7 +410,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
|
||||
NodeMap[nodeModel.ID].Node = nodeModel
|
||||
}
|
||||
nodeMapMu.Unlock()
|
||||
return err
|
||||
return fmt.Errorf("node HTTP probe failed: %w", err)
|
||||
}
|
||||
|
||||
nodeMapMu.Lock()
|
||||
@@ -377,12 +424,12 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
|
||||
|
||||
u, err := nodeModel.GetWebSocketURL("/api/analytic/intro")
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("build node WebSocket URL: %w", err)
|
||||
}
|
||||
|
||||
header := http.Header{}
|
||||
if err := nodeauth.SignWebSocketHeaders(nodeModel, u, header); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("sign node WebSocket request: %w", err)
|
||||
}
|
||||
|
||||
dial := &websocket.Dialer{
|
||||
@@ -392,7 +439,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
|
||||
|
||||
c, _, err := dial.DialContext(scopeCtx, u, header)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("connect node WebSocket: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -448,7 +495,7 @@ func nodeAnalyticRecord(nodeModel *model.Node, ctx context.Context) error {
|
||||
var rawMsg json.RawMessage
|
||||
err = c.ReadJSON(&rawMsg)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("read node WebSocket status: %w", err)
|
||||
}
|
||||
|
||||
nodeMapMu.Lock()
|
||||
|
||||
@@ -2,6 +2,7 @@ package analytic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -123,7 +124,7 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) {
|
||||
retryMutex.Unlock()
|
||||
})
|
||||
|
||||
markConnectionFailure(nodeID)
|
||||
markConnectionFailure(nodeID, context.DeadlineExceeded)
|
||||
|
||||
nodeMapMu.RLock()
|
||||
node := cloneNode(NodeMap[nodeID])
|
||||
@@ -134,6 +135,9 @@ func TestConnectionFailureKeepsFreshNodeOnline(t *testing.T) {
|
||||
if !node.ResponseAt.Equal(lastResponse) {
|
||||
t.Fatalf("expected last successful response time to be preserved, got %v", node.ResponseAt)
|
||||
}
|
||||
if node.ConnectionError != context.DeadlineExceeded.Error() || node.ConnectionErrorAt == nil {
|
||||
t.Fatalf("expected the latest connection error to be retained, got %q at %v", node.ConnectionError, node.ConnectionErrorAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
|
||||
@@ -155,7 +159,7 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
|
||||
retryMutex.Unlock()
|
||||
})
|
||||
|
||||
markConnectionFailure(nodeID)
|
||||
markConnectionFailure(nodeID, context.DeadlineExceeded)
|
||||
|
||||
nodeMapMu.RLock()
|
||||
node := cloneNode(NodeMap[nodeID])
|
||||
@@ -168,6 +172,40 @@ func TestConnectionFailureMarksStaleNodeOffline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionFailureClassifiesClockSkew(t *testing.T) {
|
||||
nodeID := uint64(50)
|
||||
now := time.Now()
|
||||
certificate := &x509.Certificate{NotBefore: now.Add(10 * time.Minute)}
|
||||
|
||||
nodeMapMu.Lock()
|
||||
NodeMap[nodeID] = &Node{}
|
||||
nodeMapMu.Unlock()
|
||||
retryMutex.Lock()
|
||||
delete(retryStates, nodeID)
|
||||
retryMutex.Unlock()
|
||||
t.Cleanup(func() {
|
||||
nodeMapMu.Lock()
|
||||
delete(NodeMap, nodeID)
|
||||
nodeMapMu.Unlock()
|
||||
retryMutex.Lock()
|
||||
delete(retryStates, nodeID)
|
||||
retryMutex.Unlock()
|
||||
})
|
||||
|
||||
markConnectionFailure(nodeID, x509.CertificateInvalidError{
|
||||
Cert: certificate,
|
||||
Reason: x509.Expired,
|
||||
Detail: "current time is before the certificate validity period",
|
||||
})
|
||||
|
||||
nodeMapMu.RLock()
|
||||
node := cloneNode(NodeMap[nodeID])
|
||||
nodeMapMu.RUnlock()
|
||||
if node.ConnectionErrorCode != NodeConnectionErrorClockSkew {
|
||||
t.Fatalf("connection error code = %q, want %q", node.ConnectionErrorCode, NodeConnectionErrorClockSkew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
|
||||
nodeID := uint64(45)
|
||||
retryMutex.Lock()
|
||||
@@ -180,8 +218,20 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
|
||||
retryMutex.Lock()
|
||||
delete(retryStates, nodeID)
|
||||
retryMutex.Unlock()
|
||||
nodeMapMu.Lock()
|
||||
delete(NodeMap, nodeID)
|
||||
nodeMapMu.Unlock()
|
||||
})
|
||||
|
||||
failedAt := time.Now()
|
||||
nodeMapMu.Lock()
|
||||
NodeMap[nodeID] = &Node{
|
||||
ConnectionError: context.DeadlineExceeded.Error(),
|
||||
ConnectionErrorCode: NodeConnectionErrorClockSkew,
|
||||
ConnectionErrorAt: &failedAt,
|
||||
}
|
||||
nodeMapMu.Unlock()
|
||||
|
||||
if !markConnectionSuccess(nodeID) {
|
||||
t.Fatal("expected a successful sample after failures to report recovery")
|
||||
}
|
||||
@@ -195,6 +245,13 @@ func TestSuccessfulSampleResetsRetryBackoff(t *testing.T) {
|
||||
if state.NextRetry.After(time.Now()) {
|
||||
t.Fatalf("expected retry to be immediately available, got %v", state.NextRetry)
|
||||
}
|
||||
nodeMapMu.RLock()
|
||||
node := cloneNode(NodeMap[nodeID])
|
||||
nodeMapMu.RUnlock()
|
||||
if node.ConnectionError != "" || node.ConnectionErrorCode != "" || node.ConnectionErrorAt != nil {
|
||||
t.Fatalf("expected a successful sample to clear the connection error, got %q (%q) at %v",
|
||||
node.ConnectionError, node.ConnectionErrorCode, node.ConnectionErrorAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) {
|
||||
@@ -208,10 +265,10 @@ func TestConnectionFailureCountIdentifiesFirstFailure(t *testing.T) {
|
||||
retryMutex.Unlock()
|
||||
})
|
||||
|
||||
if count := markConnectionFailure(nodeID); count != 1 {
|
||||
if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 1 {
|
||||
t.Fatalf("first failure count = %d, want 1", count)
|
||||
}
|
||||
if count := markConnectionFailure(nodeID); count != 2 {
|
||||
if count := markConnectionFailure(nodeID, context.DeadlineExceeded); count != 2 {
|
||||
t.Fatalf("second failure count = %d, want 2", count)
|
||||
}
|
||||
}
|
||||
|
||||
129
internal/analytic/node_time_skew_test.go
Normal file
129
internal/analytic/node_time_skew_test.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package analytic
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/0xJacky/Nginx-UI/internal/nodeauth"
|
||||
"github.com/0xJacky/Nginx-UI/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const fourSecondNodeClockSkew = 4 * time.Second
|
||||
|
||||
func TestNodeTLSHandshakeFailsWhileCertificateIsFourSecondsInFuture(t *testing.T) {
|
||||
const secret = "time-skew-node-secret"
|
||||
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Header.Get("X-Node-Secret") != secret {
|
||||
writer.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(writer).Encode(NodeInfo{Version: "time-skew-test"}); err != nil {
|
||||
t.Errorf("encode node info: %v", err)
|
||||
}
|
||||
}))
|
||||
|
||||
serverIP := server.Listener.Addr().(*net.TCPAddr).IP
|
||||
serverCertificate, rootCertificate, notBefore := newFutureNodeCertificate(t, serverIP, fourSecondNodeClockSkew)
|
||||
server.TLS = &tls.Config{
|
||||
Certificates: []tls.Certificate{serverCertificate},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
server.StartTLS()
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
node := &model.Node{
|
||||
Model: model.Model{ID: 49},
|
||||
Name: "four-second-clock-skew",
|
||||
URL: server.URL,
|
||||
}
|
||||
setupLegacyNodeAuthForTest(t, node, secret)
|
||||
|
||||
roots := x509.NewCertPool()
|
||||
roots.AddCert(rootCertificate)
|
||||
client := &http.Client{
|
||||
Transport: nodeauth.NewTransport(node, &http.Transport{TLSClientConfig: &tls.Config{
|
||||
RootCAs: roots,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}}),
|
||||
Timeout: 2 * time.Second,
|
||||
}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
|
||||
requestURL := server.URL + "/api/node"
|
||||
response, err := client.Get(requestURL)
|
||||
require.Nil(t, response)
|
||||
require.ErrorContains(t, err, "is not yet valid")
|
||||
|
||||
wait := time.Until(notBefore.Add(100 * time.Millisecond))
|
||||
require.Positive(t, wait, "test setup took longer than the four-second skew window")
|
||||
timer := time.NewTimer(wait)
|
||||
t.Cleanup(func() { timer.Stop() })
|
||||
<-timer.C
|
||||
|
||||
response, err = client.Get(requestURL)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { response.Body.Close() })
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
}
|
||||
|
||||
func newFutureNodeCertificate(t *testing.T, serverIP net.IP, futureOffset time.Duration) (
|
||||
tls.Certificate,
|
||||
*x509.Certificate,
|
||||
time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
rootPublicKey, rootPrivateKey, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
rootTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "Nginx UI node time-skew test root"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageCertSign,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
}
|
||||
rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, rootPublicKey, rootPrivateKey)
|
||||
require.NoError(t, err)
|
||||
rootCertificate, err := x509.ParseCertificate(rootDER)
|
||||
require.NoError(t, err)
|
||||
|
||||
leafPublicKey, leafPrivateKey, err := ed25519.GenerateKey(rand.Reader)
|
||||
require.NoError(t, err)
|
||||
notBefore := time.Now().Add(futureOffset).Truncate(time.Second)
|
||||
leafTemplate := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(2),
|
||||
Subject: pkix.Name{CommonName: "Nginx UI child node"},
|
||||
IPAddresses: []net.IP{serverIP},
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notBefore.Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
}
|
||||
leafDER, err := x509.CreateCertificate(rand.Reader, leafTemplate, rootTemplate, leafPublicKey, rootPrivateKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER})
|
||||
privateKeyDER, err := x509.MarshalPKCS8PrivateKey(leafPrivateKey)
|
||||
require.NoError(t, err)
|
||||
privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER})
|
||||
serverCertificate, err := tls.X509KeyPair(certificatePEM, privateKeyPEM)
|
||||
require.NoError(t, err)
|
||||
|
||||
return serverCertificate, rootCertificate, notBefore
|
||||
}
|
||||
Reference in New Issue
Block a user