feat(node): show authentication upgrade progress

This commit is contained in:
0xJacky
2026-08-17 19:11:00 +08:00
parent 7cbc4db8cf
commit ffc29affca
12 changed files with 928 additions and 70 deletions

View File

@@ -31,19 +31,27 @@ type nodeMutationRequest struct {
}
type nodeResponse struct {
ID uint64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
URL string `json:"url"`
Enabled bool `json:"enabled"`
AuthMethod string `json:"auth_method"`
HasCredential bool `json:"has_credential"`
CredentialStatus string `json:"credential_status"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
ConnectionError string `json:"connection_error,omitempty"`
ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
ID uint64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
URL string `json:"url"`
Enabled bool `json:"enabled"`
AuthMethod string `json:"auth_method"`
HasCredential bool `json:"has_credential"`
CredentialStatus string `json:"credential_status"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
AuthUpgradeStatus string `json:"auth_upgrade_status,omitempty"`
AuthUpgradeStep string `json:"auth_upgrade_step,omitempty"`
AuthUpgradeAttemptCount uint `json:"auth_upgrade_attempt_count"`
AuthUpgradeAttemptedAt *time.Time `json:"auth_upgrade_attempted_at,omitempty"`
AuthUpgradeNextRetryAt *time.Time `json:"auth_upgrade_next_retry_at,omitempty"`
AuthUpgradeCompletedAt *time.Time `json:"auth_upgrade_completed_at,omitempty"`
AuthUpgradeErrorCode string `json:"auth_upgrade_error_code,omitempty"`
AuthUpgradeError string `json:"auth_upgrade_error,omitempty"`
ConnectionError string `json:"connection_error,omitempty"`
ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"`
ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"`
// LegacySecret only ever carries the redaction sentinel, which tells the
// edit form a secret is stored without putting it in a list response.
LegacySecret string `json:"legacy_secret,omitempty"`
@@ -53,17 +61,39 @@ type nodeResponse struct {
func newNodeResponse(node *model.Node) nodeResponse {
analyticNode := analytic.GetNode(node)
authUpgradeStatus := node.AuthUpgradeStatus
authUpgradeErrorCode := node.AuthUpgradeErrorCode
authUpgradeError := node.AuthUpgradeError
if node.AuthMethod == model.NodeAuthMethodLegacy && authUpgradeStatus == "" {
authUpgradeStatus = model.NodeAuthUpgradeStatusPending
}
if node.AuthMethod == model.NodeAuthMethodLegacy && len(node.EncryptedLegacySecret) == 0 {
authUpgradeStatus = model.NodeAuthUpgradeStatusFailed
authUpgradeErrorCode = model.NodeAuthUpgradeErrorMissingLegacySecret
authUpgradeError = "The stored legacy node secret is unavailable."
}
if node.AuthMethod == model.NodeAuthMethodLegacy && !node.Enabled {
authUpgradeStatus = model.NodeAuthUpgradeStatusPaused
}
response := nodeResponse{
ID: node.ID,
CreatedAt: node.CreatedAt,
UpdatedAt: node.UpdatedAt,
Name: node.Name,
URL: node.URL,
Enabled: node.Enabled,
AuthMethod: node.AuthMethod,
HasCredential: node.HasCredential(),
CredentialStatus: node.CredentialStatus,
LastCredentialUseAt: node.LastCredentialUseAt,
ID: node.ID,
CreatedAt: node.CreatedAt,
UpdatedAt: node.UpdatedAt,
Name: node.Name,
URL: node.URL,
Enabled: node.Enabled,
AuthMethod: node.AuthMethod,
HasCredential: node.HasCredential(),
CredentialStatus: node.CredentialStatus,
LastCredentialUseAt: node.LastCredentialUseAt,
AuthUpgradeStatus: authUpgradeStatus,
AuthUpgradeStep: node.AuthUpgradeStep,
AuthUpgradeAttemptCount: node.AuthUpgradeAttemptCount,
AuthUpgradeAttemptedAt: node.AuthUpgradeAttemptedAt,
AuthUpgradeNextRetryAt: node.AuthUpgradeNextRetryAt,
AuthUpgradeCompletedAt: node.AuthUpgradeCompletedAt,
AuthUpgradeErrorCode: authUpgradeErrorCode,
AuthUpgradeError: authUpgradeError,
}
if len(node.EncryptedLegacySecret) != 0 {
response.LegacySecret = settings.RedactedSensitiveValue
@@ -113,16 +143,22 @@ func AddNode(c *gin.Context) {
legacySecret := mutationLegacySecret(request)
authMethod := model.NodeAuthMethodPaired
credentialStatus := model.NodeCredentialStatusUnpaired
authUpgradeStatus := ""
authUpgradeStep := ""
if legacySecret != "" {
authMethod = model.NodeAuthMethodLegacy
credentialStatus = model.NodeCredentialStatusActive
authUpgradeStatus = model.NodeAuthUpgradeStatusPending
authUpgradeStep = model.NodeAuthUpgradeStepQueued
}
node := &model.Node{
Name: request.Name,
URL: normalizedURL,
Enabled: request.Enabled,
AuthMethod: authMethod,
CredentialStatus: credentialStatus,
Name: request.Name,
URL: normalizedURL,
Enabled: request.Enabled,
AuthMethod: authMethod,
CredentialStatus: credentialStatus,
AuthUpgradeStatus: authUpgradeStatus,
AuthUpgradeStep: authUpgradeStep,
}
database := model.UseDB()
err = database.Transaction(func(tx *gorm.DB) error {
@@ -147,6 +183,9 @@ func AddNode(c *gin.Context) {
return
}
refreshNodeState()
if node.Enabled && legacySecret != "" {
nodeauth.QueueLegacyRelationshipUpgrade(node.ID)
}
c.JSON(http.StatusCreated, newNodeResponse(node))
}
@@ -185,6 +224,12 @@ func EditNode(c *gin.Context) {
updates["encrypted_legacy_secret"] = encrypted
updates["auth_method"] = model.NodeAuthMethodLegacy
updates["credential_status"] = model.NodeCredentialStatusActive
updates["auth_upgrade_status"] = model.NodeAuthUpgradeStatusPending
updates["auth_upgrade_step"] = model.NodeAuthUpgradeStepQueued
updates["auth_upgrade_next_retry_at"] = time.Now()
updates["auth_upgrade_completed_at"] = nil
updates["auth_upgrade_error_code"] = ""
updates["auth_upgrade_error"] = ""
if err := tx.Unscoped().Where("node_id = ?", node.ID).Delete(&model.NodeCredential{}).Error; err != nil {
return err
}
@@ -200,9 +245,33 @@ func EditNode(c *gin.Context) {
return
}
refreshNodeState()
if node.Enabled && node.AuthMethod == model.NodeAuthMethodLegacy && len(node.EncryptedLegacySecret) != 0 {
nodeauth.QueueLegacyRelationshipUpgrade(node.ID)
}
c.JSON(http.StatusOK, newNodeResponse(node))
}
func RetryNodeAuthUpgrade(c *gin.Context) {
node, ok := findNode(c, false)
if !ok {
return
}
if err := nodeauth.RetryLegacyRelationshipUpgrade(node.ID, time.Now()); err != nil {
status := http.StatusInternalServerError
if errors.Is(err, nodeauth.ErrRelationshipUpgradeAlreadyRunning) ||
errors.Is(err, nodeauth.ErrRelationshipUpgradeNotAvailable) {
status = http.StatusConflict
}
c.JSON(status, gin.H{"message": err.Error()})
return
}
if err := model.UseDB().First(node, node.ID).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()})
return
}
c.JSON(http.StatusAccepted, newNodeResponse(node))
}
func DeleteNode(c *gin.Context) {
permanent := cast.ToBool(c.Query("permanent"))
node, ok := findNode(c, permanent)

View File

@@ -42,6 +42,7 @@ func TestNodeResponseRedactsAllCredentialMaterial(t *testing.T) {
require.NotContains(t, response, "token")
require.NotContains(t, response, "private_key")
require.Contains(t, response, `"auth_method":"legacy_secret"`)
require.Contains(t, response, `"auth_upgrade_status":"paused"`)
require.Contains(t, response, `"has_credential":true`)
require.Contains(t, response, `"status":false`)
require.NotContains(t, response, `"NodeStat"`)
@@ -218,3 +219,16 @@ func TestNodeRouterRegistersRecoveryRoute(t *testing.T) {
}
t.Fatal("PATCH /api/nodes/:id recovery route is not registered")
}
func TestNodeRouterRegistersAuthenticationUpgradeRetryRoute(t *testing.T) {
gin.SetMode(gin.TestMode)
router := gin.New()
InitRouter(router.Group("/api"))
for _, route := range router.Routes() {
if route.Method == http.MethodPost && route.Path == "/api/nodes/:id/auth-upgrade/retry" {
return
}
}
t.Fatal("POST /api/nodes/:id/auth-upgrade/retry is not registered")
}

View File

@@ -28,6 +28,7 @@ func InitRouter(r *gin.RouterGroup) {
admin.GET("nodes/:id/secret", GetNodeSecret)
admin.GET("nodes/:id/credentials", GetNodeCredentials)
admin.POST("nodes/:id/credentials/rotate", RotateNodeCredential)
admin.POST("nodes/:id/auth-upgrade/retry", RetryNodeAuthUpgrade)
admin.GET("node/credentials", ListControllerCredentials)
admin.DELETE("node/credentials/:credential_id", RevokeControllerCredential)
}

View File

@@ -13,6 +13,14 @@ export interface Node extends ModelBase {
has_credential: boolean
credential_status: 'active' | 'unpaired' | 'rotating' | 'revoked'
last_credential_use_at?: string
auth_upgrade_status?: 'pending' | 'in_progress' | 'waiting_target' | 'failed' | 'completed' | 'paused'
auth_upgrade_step?: 'queued' | 'request' | 'verify' | 'persist' | 'completed'
auth_upgrade_attempt_count: number
auth_upgrade_attempted_at?: string
auth_upgrade_next_retry_at?: string
auth_upgrade_completed_at?: string
auth_upgrade_error_code?: 'target_unsupported' | 'timeout' | 'connection_failed' | 'authentication_rejected' | 'target_rejected' | 'invalid_response' | 'invalid_confirmation' | 'persistence_failed' | 'missing_legacy_secret' | 'internal'
auth_upgrade_error?: string
response_at?: string
connection_error?: string
connection_error_code?: 'clock_skew'
@@ -90,6 +98,7 @@ const nodeApi = extendCurdApi(useCurdApi<Node>(baseUrl), {
syncConfigs,
getSecret: (id: number) => http.get<{ value: string }>(`${baseUrl}/${id}/secret`),
rotateCredential: (id: number) => http.post(`${baseUrl}/${id}/credentials/rotate`),
retryAuthUpgrade: (id: number) => http.post<Node>(`${baseUrl}/${id}/auth-upgrade/retry`),
})
export default nodeApi

View File

@@ -0,0 +1,268 @@
<script setup lang="ts">
import type { Node } from '@/api/node'
import {
CheckCircleOutlined,
ClockCircleOutlined,
ExclamationCircleOutlined,
LoadingOutlined,
PauseCircleOutlined,
} from '@ant-design/icons-vue'
import nodeApi from '@/api/node'
import { formatDateTime } from '@/lib/helper'
const props = defineProps<{
node: Node
}>()
const { message } = useGlobalApp()
const isRetrying = ref(false)
const retryState = shallowRef<Node>()
const nodeState = computed(() => retryState.value || props.node)
watch(() => props.node.auth_upgrade_status, status => {
if (retryState.value?.auth_upgrade_status === status)
retryState.value = undefined
})
const effectiveStatus = computed(() => {
if (nodeState.value.auth_method === 'paired_ed25519')
return nodeState.value.credential_status
if (!nodeState.value.enabled)
return 'paused'
return nodeState.value.auth_upgrade_status || 'pending'
})
const statusPresentation = computed(() => {
switch (effectiveStatus.value) {
case 'active':
return { color: 'green', label: $gettext('Paired signature'), icon: CheckCircleOutlined }
case 'rotating':
return { color: 'blue', label: $gettext('Rotating'), icon: LoadingOutlined }
case 'unpaired':
return { color: 'default', label: $gettext('Unpaired'), icon: ClockCircleOutlined }
case 'revoked':
return { color: 'red', label: $gettext('Revoked'), icon: ExclamationCircleOutlined }
case 'in_progress':
return { color: 'blue', label: $gettext('Upgrading'), icon: LoadingOutlined }
case 'waiting_target':
return { color: 'orange', label: $gettext('Waiting for target'), icon: ClockCircleOutlined }
case 'failed':
return { color: 'red', label: $gettext('Upgrade failed'), icon: ExclamationCircleOutlined }
case 'paused':
return { color: 'default', label: $gettext('Upgrade paused'), icon: PauseCircleOutlined }
default:
return { color: 'blue', label: $gettext('Upgrade pending'), icon: ClockCircleOutlined }
}
})
const statusTitle = computed(() => {
switch (effectiveStatus.value) {
case 'in_progress':
return $gettext('Authentication upgrade in progress')
case 'waiting_target':
return $gettext('Waiting for target node upgrade')
case 'failed':
return $gettext('Authentication upgrade failed')
case 'paused':
return $gettext('Authentication upgrade paused')
default:
return $gettext('Authentication upgrade pending')
}
})
const isLegacyUpgrade = computed(() => nodeState.value.auth_method !== 'paired_ed25519')
const upgradeErrorMessage = computed(() => {
switch (nodeState.value.auth_upgrade_error_code) {
case 'timeout':
return $gettext('The target node did not respond before the authentication upgrade timed out.')
case 'connection_failed':
return $gettext('The target node could not be reached. Check the node URL, network, and TLS settings.')
case 'authentication_rejected':
return $gettext('The target node rejected the saved Node Secret. Check and save the correct secret before retrying.')
case 'target_rejected':
return $gettext('The target node rejected the authentication upgrade request. Check the target node logs before retrying.')
case 'invalid_response':
return $gettext('The target node returned an invalid pairing response.')
case 'invalid_confirmation':
return $gettext('The target returned an invalid upgrade confirmation. Nginx UI stopped before accepting the new credential.')
case 'persistence_failed':
return $gettext('The paired credential could not be saved on this Nginx UI instance.')
case 'missing_legacy_secret':
return $gettext('The saved legacy Node Secret is unavailable. Save the Node Secret again before retrying.')
default:
return nodeState.value.auth_upgrade_error || $gettext('The authentication upgrade failed because of an internal error.')
}
})
const statusDescription = computed(() => {
switch (effectiveStatus.value) {
case 'in_progress':
return $gettext('The node is still connected with the legacy secret while Nginx UI switches this relationship to paired signatures.')
case 'waiting_target':
return $gettext('The target node does not support paired signatures yet. Upgrade the target node and Nginx UI will retry automatically.')
case 'failed':
return upgradeErrorMessage.value
case 'paused':
return $gettext('Enable the node to resume the authentication upgrade. The saved relationship has not been changed.')
default:
return $gettext('The legacy secret is saved and the authentication upgrade is queued. The current node connection remains available.')
}
})
const stepItems = computed(() => [
{ title: $gettext('Legacy connection ready') },
{ title: $gettext('Request paired credential') },
{ title: $gettext('Verify target confirmation') },
{ title: $gettext('Save and switch authentication') },
])
const currentStep = computed(() => {
switch (nodeState.value.auth_upgrade_step) {
case 'verify':
return 2
case 'persist':
case 'completed':
return 3
case 'request':
return 1
default:
return 1
}
})
const stepStatus = computed<'error' | 'process' | 'wait'>(() => {
if (effectiveStatus.value === 'failed')
return 'error'
if (effectiveStatus.value === 'pending' || effectiveStatus.value === 'paused' || effectiveStatus.value === 'waiting_target')
return 'wait'
return 'process'
})
const canRetry = computed(() => effectiveStatus.value === 'failed' || effectiveStatus.value === 'waiting_target')
async function retryAuthenticationUpgrade() {
isRetrying.value = true
try {
const updated = await nodeApi.retryAuthUpgrade(props.node.id)
retryState.value = updated
message.success($gettext('Authentication upgrade queued'))
}
finally {
isRetrying.value = false
}
}
</script>
<template>
<ATag v-if="!isLegacyUpgrade" :color="statusPresentation.color" class="m-0">
<component
:is="statusPresentation.icon"
:class="{ 'auth-upgrade-spinner': effectiveStatus === 'rotating' }"
/>
{{ statusPresentation.label }}
</ATag>
<APopover v-else placement="rightTop" trigger="click">
<template #title>
<div class="flex items-center gap-2">
<component
:is="statusPresentation.icon"
:class="{ 'auth-upgrade-spinner': effectiveStatus === 'in_progress' }"
/>
<span>{{ statusTitle }}</span>
</div>
</template>
<template #content>
<div class="w-96 max-w-[calc(100vw-48px)]">
<AAlert
:type="effectiveStatus === 'failed' ? 'error' : effectiveStatus === 'waiting_target' ? 'warning' : 'info'"
:message="statusDescription"
show-icon
class="mb-4"
/>
<ASteps
direction="vertical"
size="small"
:current="currentStep"
:status="stepStatus"
:items="stepItems"
/>
<dl v-if="nodeState.auth_upgrade_attempted_at || nodeState.auth_upgrade_next_retry_at" class="mb-0 mt-3 grid grid-cols-[auto_1fr] gap-x-3 gap-y-1 text-sm">
<template v-if="nodeState.auth_upgrade_attempted_at">
<dt class="text-gray-500 dark:text-gray-400">
{{ $gettext('Last attempt') }}
</dt>
<dd class="m-0">
{{ formatDateTime(nodeState.auth_upgrade_attempted_at) }}
</dd>
</template>
<template v-if="nodeState.auth_upgrade_next_retry_at && effectiveStatus !== 'paused'">
<dt class="text-gray-500 dark:text-gray-400">
{{ $gettext('Automatic retry after') }}
</dt>
<dd class="m-0">
{{ formatDateTime(nodeState.auth_upgrade_next_retry_at) }}
</dd>
</template>
<template v-if="nodeState.auth_upgrade_attempt_count">
<dt class="text-gray-500 dark:text-gray-400">
{{ $gettext('Attempts') }}
</dt>
<dd class="m-0">
{{ nodeState.auth_upgrade_attempt_count }}
</dd>
</template>
</dl>
<details v-if="nodeState.auth_upgrade_error_code && (effectiveStatus === 'failed' || effectiveStatus === 'waiting_target')" class="mt-3 text-sm">
<summary class="cursor-pointer select-none font-medium">
{{ $gettext('Technical details') }}
</summary>
<code class="mt-2 block rounded bg-gray-100 px-3 py-2 text-xs dark:bg-gray-800">
{{ nodeState.auth_upgrade_error_code }}
</code>
</details>
<AButton
v-if="canRetry"
type="primary"
size="small"
class="mt-4"
:loading="isRetrying"
@click="retryAuthenticationUpgrade"
>
{{ $gettext('Retry authentication upgrade') }}
</AButton>
</div>
</template>
<button
type="button"
class="cursor-pointer border-0 bg-transparent p-0 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-500"
:aria-label="statusTitle"
>
<ATag :color="statusPresentation.color" class="m-0">
<component
:is="statusPresentation.icon"
:class="{ 'auth-upgrade-spinner': effectiveStatus === 'in_progress' }"
/>
{{ statusPresentation.label }}
</ATag>
</button>
</APopover>
</template>
<style scoped>
.auth-upgrade-spinner {
animation: loadingCircle 1s infinite linear;
}
@media (prefers-reduced-motion: reduce) {
.auth-upgrade-spinner {
animation: none;
}
}
</style>

View File

@@ -7,14 +7,7 @@ import { Badge, InputPassword, Popover, Tag } from 'ant-design-vue'
import { h } from 'vue'
import nodeApi from '@/api/node'
import { SensitiveInput } from '@/components/SensitiveString'
// Only a credential that is not simply healthy is worth its own word: the
// remaining states collapse into naming the authentication method itself.
const unhealthyCredentialMap: Record<string, { color: string, text: () => string }> = {
rotating: { color: 'blue', text: () => $gettext('Rotating') },
unpaired: { color: 'default', text: () => $gettext('Unpaired') },
revoked: { color: 'red', text: () => $gettext('Revoked') },
}
import NodeAuthStatus from './NodeAuthStatus.vue'
function renderConnectionErrorContent(record: Node) {
if (record.connection_error_code !== 'clock_skew') {
@@ -142,18 +135,9 @@ const columns: StdTableColumn[] = [{
}, {
title: () => $gettext('Authentication'),
dataIndex: 'auth_method',
customRender: ({ record }: CustomRenderArgs) => {
if (record.auth_method !== 'paired_ed25519')
return <Tag color="orange" class="m-0">{$gettext('Legacy secret')}</Tag>
const unhealthy = unhealthyCredentialMap[record.credential_status as string]
if (unhealthy)
return <Tag color={unhealthy.color} class="m-0">{unhealthy.text()}</Tag>
return <Tag color="green" class="m-0">{$gettext('Paired signature')}</Tag>
},
customRender: ({ record }: CustomRenderArgs) => <NodeAuthStatus node={record as Node} />,
pure: true,
width: 140,
width: 160,
}, {
title: () => $gettext('Status'),
dataIndex: 'status',

View File

@@ -0,0 +1,131 @@
import { expect, test } from '@playwright/test'
import { expectTableRows, gotoRoute } from './helpers'
test('node list explains authentication upgrade progress and failure', async ({ page }, testInfo) => {
await page.setViewportSize({ width: 1600, height: 1000 })
await page.route('**/api/self_check', async route => {
await route.fulfill({ status: 200, contentType: 'application/json', json: [] })
})
await page.route('**/api/nodes?**', async route => {
if (route.request().method() !== 'GET') {
await route.continue()
return
}
const fixture = {
created_at: '2026-08-17T08:00:00Z',
updated_at: '2026-08-17T09:00:00Z',
url: 'https://node.example.test:9000',
version: 'v2.5.9',
status: true,
enabled: true,
auth_method: 'legacy_secret',
credential_status: 'active',
has_credential: true,
auth_upgrade_attempt_count: 0,
}
const data = [
{
...fixture,
id: 9101,
name: 'edge-pending',
status: true,
enabled: true,
auth_method: 'legacy_secret',
credential_status: 'active',
auth_upgrade_status: 'pending',
auth_upgrade_step: 'queued',
auth_upgrade_attempt_count: 0,
},
{
...fixture,
id: 9102,
name: 'edge-upgrading',
status: true,
enabled: true,
auth_method: 'legacy_secret',
credential_status: 'active',
auth_upgrade_status: 'in_progress',
auth_upgrade_step: 'verify',
auth_upgrade_attempt_count: 1,
auth_upgrade_attempted_at: '2026-08-17T09:22:10Z',
},
{
...fixture,
id: 9103,
name: 'edge-waiting-target',
status: true,
enabled: true,
auth_method: 'legacy_secret',
credential_status: 'active',
auth_upgrade_status: 'waiting_target',
auth_upgrade_step: 'request',
auth_upgrade_attempt_count: 1,
auth_upgrade_attempted_at: '2026-08-17T09:18:32Z',
auth_upgrade_next_retry_at: '2026-08-17T10:18:32Z',
auth_upgrade_error_code: 'target_unsupported',
},
{
...fixture,
id: 9104,
name: 'edge-upgrade-failed',
status: true,
enabled: true,
auth_method: 'legacy_secret',
credential_status: 'active',
auth_upgrade_status: 'failed',
auth_upgrade_step: 'verify',
auth_upgrade_attempt_count: 2,
auth_upgrade_attempted_at: '2026-08-17T09:15:06Z',
auth_upgrade_next_retry_at: '2026-08-17T10:15:06Z',
auth_upgrade_error_code: 'invalid_confirmation',
auth_upgrade_error: 'The target node returned an invalid upgrade confirmation.',
},
]
await route.fulfill({
status: 200,
contentType: 'application/json',
json: {
data,
pagination: {
total: data.length,
per_page: 20,
current_page: 1,
total_pages: 1,
},
},
})
})
await gotoRoute(page, '/nodes')
const rows = await expectTableRows(page, 4)
await expect(rows.filter({ hasText: 'edge-pending' })).toContainText('Upgrade pending')
await expect(rows.filter({ hasText: 'edge-upgrading' })).toContainText('Upgrading')
await expect(rows.filter({ hasText: 'edge-waiting-target' })).toContainText('Waiting for target')
await expect(rows.filter({ hasText: 'edge-upgrade-failed' })).toContainText('Upgrade failed')
await page.screenshot({
path: testInfo.outputPath('node-auth-upgrade-overview.png'),
fullPage: true,
})
const failedRow = rows.filter({ hasText: 'edge-upgrade-failed' })
await failedRow.getByRole('button', { name: 'Authentication upgrade failed' }).click()
const popover = page.locator('.ant-popover').filter({ hasText: 'Authentication upgrade failed' })
await expect(popover).toBeVisible()
await expect(popover).toContainText('The target returned an invalid upgrade confirmation.')
await expect(popover).toContainText('Verify target confirmation')
await expect(popover).toContainText('Retry authentication upgrade')
await expect(popover).toContainText('invalid_confirmation')
await popover.getByText('Technical details').click()
await expect(popover.getByText('invalid_confirmation')).toBeVisible()
await page.screenshot({
path: testInfo.outputPath('node-auth-upgrade-failed.png'),
fullPage: true,
})
})

View File

@@ -4,6 +4,8 @@ import (
"context"
"github.com/0xJacky/Nginx-UI/internal/cert"
"github.com/0xJacky/Nginx-UI/internal/nodeauth"
"github.com/0xJacky/Nginx-UI/settings"
"github.com/go-co-op/gocron/v2"
"github.com/uozi-tech/cosy/logger"
)
@@ -55,6 +57,7 @@ func InitCronJobs(ctx context.Context) {
}
// Initialize automatic node credential upgrade and rotation.
nodeauth.StartRelationshipUpgradeWorker(ctx, settings.NodeSettings.InstanceID)
_, err = setupNodeCredentialMaintenanceJob(s)
if err != nil {
logger.Fatalf("NodeCredentialMaintenance Err: %v\n", err)

View File

@@ -54,6 +54,29 @@ type MaintenanceIssue struct {
Err error
}
type relationshipUpgradeError struct {
Step string
Err error
}
func (e *relationshipUpgradeError) Error() string {
return e.Err.Error()
}
func (e *relationshipUpgradeError) Unwrap() error {
return e.Err
}
type relationshipHTTPError struct {
StatusCode int
Status string
Message string
}
func (e *relationshipHTTPError) Error() string {
return fmt.Sprintf("node returned %s: %s", e.Status, e.Message)
}
type upgradeRequest struct {
ControllerInstanceID string `json:"controller_instance_id"`
PublicKey string `json:"public_key"`
@@ -77,40 +100,62 @@ type rotationRequest struct {
// before the new credential is stored, so a substituted response cannot leave
// the relationship relying on a key the real target never issued.
func UpgradeLegacyRelationship(ctx context.Context, node *model.Node, controllerInstanceID string) (*PairingResult, error) {
return upgradeLegacyRelationship(ctx, node, controllerInstanceID, nil)
}
func upgradeLegacyRelationship(ctx context.Context, node *model.Node, controllerInstanceID string,
reportStep func(string),
) (*PairingResult, error) {
if node == nil {
return nil, errors.New("node is required")
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: errors.New("node is required")}
}
if len(node.EncryptedLegacySecret) == 0 {
return nil, ErrLegacySecretMissing
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: ErrLegacySecretMissing}
}
if _, err := uuid.Parse(controllerInstanceID); err != nil {
return nil, errors.New("invalid controller instance ID")
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: errors.New("invalid controller instance ID")}
}
secret, err := DecryptPrivateCredential(LegacyCredentialPurpose(node.ID), node.EncryptedLegacySecret)
if err != nil {
return nil, err
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err}
}
publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return nil, err
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err}
}
reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepRequest)
var response pairingResponse
if err := sendRelationshipJSON(ctx, node, http.MethodPost, "/api/node/pair/upgrade", upgradeRequest{
ControllerInstanceID: controllerInstanceID,
PublicKey: base64.RawURLEncoding.EncodeToString(publicKey),
}, &response, true); err != nil {
return nil, err
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err}
}
reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepVerify)
if err := validatePairingResponse(&response); err != nil {
return nil, err
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepVerify, Err: err}
}
if err := VerifyUpgradeConfirmation(secret, controllerInstanceID, publicKey,
response.CredentialID, response.TargetInstanceID, response.Confirmation); err != nil {
return nil, fmt.Errorf("target node did not confirm the upgrade: %w", err)
return nil, &relationshipUpgradeError{
Step: model.NodeAuthUpgradeStepVerify,
Err: fmt.Errorf("target node did not confirm the upgrade: %w", err),
}
}
reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepPersist)
result, err := persistRelationship(node, &response, publicKey, privateKey)
if err != nil {
return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepPersist, Err: err}
}
return result, nil
}
func reportRelationshipUpgradeStep(reportStep func(string), step string) {
if reportStep != nil {
reportStep(step)
}
return persistRelationship(node, &response, publicKey, privateKey)
}
func validatePairingResponse(response *pairingResponse) error {
@@ -157,10 +202,17 @@ func persistRelationship(node *model.Node, response *pairingResponse,
// node that loses its controller credential (a restored backup, a
// revoked relationship) be re-upgraded automatically instead of waiting
// for someone to notice and pair it by hand.
now := time.Now()
return tx.Model(&model.Node{}).Where("id = ?", node.ID).Updates(map[string]any{
"token": "",
"auth_method": model.NodeAuthMethodPaired,
"credential_status": model.NodeCredentialStatusActive,
"token": "",
"auth_method": model.NodeAuthMethodPaired,
"credential_status": model.NodeCredentialStatusActive,
"auth_upgrade_status": model.NodeAuthUpgradeStatusCompleted,
"auth_upgrade_step": model.NodeAuthUpgradeStepCompleted,
"auth_upgrade_completed_at": now,
"auth_upgrade_next_retry_at": nil,
"auth_upgrade_error_code": "",
"auth_upgrade_error": "",
}).Error
})
if err != nil {
@@ -281,11 +333,14 @@ func MaintainRelationships(ctx context.Context, controllerInstanceID string, now
if len(node.EncryptedLegacySecret) == 0 {
continue
}
if node.AuthUpgradeNextRetryAt != nil && node.AuthUpgradeNextRetryAt.After(now) {
continue
}
// A node that has not been upgraded yet simply keeps using the shared
// secret. That is an expected state during a rolling upgrade, not a
// fault worth reporting on every pass.
if _, err := UpgradeLegacyRelationship(ctx, node, controllerInstanceID); err != nil &&
!errors.Is(err, ErrRelationshipUnsupported) {
if err := RunLegacyRelationshipUpgrade(ctx, node.ID, controllerInstanceID, now); err != nil &&
!errors.Is(err, ErrRelationshipUpgradeAlreadyRunning) {
issues = append(issues, MaintenanceIssue{NodeID: node.ID, Operation: "upgrade", Err: err})
}
case model.NodeAuthMethodPaired:
@@ -366,7 +421,11 @@ func sendRelationshipJSON(ctx context.Context, node *model.Node, method, path st
if message == "" {
message = http.StatusText(httpResponse.StatusCode)
}
return fmt.Errorf("node returned %s: %s", httpResponse.Status, message)
return &relationshipHTTPError{
StatusCode: httpResponse.StatusCode,
Status: httpResponse.Status,
Message: message,
}
}
if response != nil && len(responseBody) != 0 {
if err := json.Unmarshal(responseBody, response); err != nil {

View File

@@ -0,0 +1,225 @@
package nodeauth
import (
"context"
"encoding/json"
"errors"
"net"
"net/http"
"sync"
"time"
"github.com/0xJacky/Nginx-UI/model"
"github.com/uozi-tech/cosy/logger"
"gorm.io/gorm"
)
const (
relationshipUpgradeRetryDelay = time.Hour
relationshipUpgradeStaleAfter = 10 * time.Minute
relationshipUpgradeQueueSize = 128
)
var (
ErrRelationshipUpgradeAlreadyRunning = errors.New("node authentication upgrade is already running")
ErrRelationshipUpgradeNotAvailable = errors.New("node authentication upgrade is not available")
relationshipUpgradeQueue = make(chan uint64, relationshipUpgradeQueueSize)
relationshipUpgradeWorkerOnce sync.Once
)
type authUpgradeFailure struct {
Step string
Code string
Message string
}
func StartRelationshipUpgradeWorker(ctx context.Context, controllerInstanceID string) {
relationshipUpgradeWorkerOnce.Do(func() {
go func() {
for {
select {
case <-ctx.Done():
return
case nodeID := <-relationshipUpgradeQueue:
attemptCtx, cancel := context.WithTimeout(ctx, time.Minute)
err := RunLegacyRelationshipUpgrade(attemptCtx, nodeID, controllerInstanceID, time.Now())
cancel()
if err != nil && !errors.Is(err, ErrRelationshipUpgradeAlreadyRunning) {
logger.Warnf("Automatic node authentication upgrade failed for node %d: %v", nodeID, err)
}
}
}
}()
})
}
func QueueLegacyRelationshipUpgrade(nodeID uint64) bool {
select {
case relationshipUpgradeQueue <- nodeID:
return true
default:
return false
}
}
func RunLegacyRelationshipUpgrade(ctx context.Context, nodeID uint64, controllerInstanceID string,
now time.Time,
) error {
database := model.UseDB()
if database == nil {
return errors.New("node authentication database is unavailable")
}
var node model.Node
if err := database.First(&node, nodeID).Error; err != nil {
return err
}
if node.AuthMethod != model.NodeAuthMethodLegacy || !node.Enabled || len(node.EncryptedLegacySecret) == 0 {
return ErrRelationshipUpgradeNotAvailable
}
staleBefore := now.Add(-relationshipUpgradeStaleAfter)
result := database.Model(&model.Node{}).
Where("id = ? AND auth_method = ?", nodeID, model.NodeAuthMethodLegacy).
Where("auth_upgrade_status IS NULL OR auth_upgrade_status <> ? OR auth_upgrade_attempted_at IS NULL OR auth_upgrade_attempted_at < ?",
model.NodeAuthUpgradeStatusInProgress, staleBefore).
Updates(map[string]any{
"auth_upgrade_status": model.NodeAuthUpgradeStatusInProgress,
"auth_upgrade_step": model.NodeAuthUpgradeStepRequest,
"auth_upgrade_attempt_count": gorm.Expr("auth_upgrade_attempt_count + 1"),
"auth_upgrade_attempted_at": now,
"auth_upgrade_next_retry_at": nil,
"auth_upgrade_error_code": "",
"auth_upgrade_error": "",
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return ErrRelationshipUpgradeAlreadyRunning
}
reportStep := func(step string) {
_ = database.Model(&model.Node{}).
Where("id = ? AND auth_upgrade_status = ?", nodeID, model.NodeAuthUpgradeStatusInProgress).
Update("auth_upgrade_step", step).Error
}
_, err := upgradeLegacyRelationship(ctx, &node, controllerInstanceID, reportStep)
if err == nil {
return nil
}
failure := classifyAuthUpgradeFailure(err)
nextRetryAt := now.Add(relationshipUpgradeRetryDelay)
status := model.NodeAuthUpgradeStatusFailed
if failure.Code == model.NodeAuthUpgradeErrorTargetUnsupported {
status = model.NodeAuthUpgradeStatusWaitingTarget
}
if updateErr := database.Model(&model.Node{}).Where("id = ?", nodeID).Updates(map[string]any{
"auth_upgrade_status": status,
"auth_upgrade_step": failure.Step,
"auth_upgrade_next_retry_at": nextRetryAt,
"auth_upgrade_error_code": failure.Code,
"auth_upgrade_error": failure.Message,
}).Error; updateErr != nil {
return errors.Join(err, updateErr)
}
if status == model.NodeAuthUpgradeStatusWaitingTarget {
return nil
}
return err
}
func RetryLegacyRelationshipUpgrade(nodeID uint64, now time.Time) error {
database := model.UseDB()
if database == nil {
return errors.New("node authentication database is unavailable")
}
var node model.Node
if err := database.First(&node, nodeID).Error; err != nil {
return err
}
if node.AuthMethod != model.NodeAuthMethodLegacy || !node.Enabled || len(node.EncryptedLegacySecret) == 0 {
return ErrRelationshipUpgradeNotAvailable
}
if node.AuthUpgradeStatus == model.NodeAuthUpgradeStatusInProgress && node.AuthUpgradeAttemptedAt != nil &&
node.AuthUpgradeAttemptedAt.After(now.Add(-relationshipUpgradeStaleAfter)) {
return ErrRelationshipUpgradeAlreadyRunning
}
if err := database.Model(&node).Updates(map[string]any{
"auth_upgrade_status": model.NodeAuthUpgradeStatusPending,
"auth_upgrade_step": model.NodeAuthUpgradeStepQueued,
"auth_upgrade_next_retry_at": now,
"auth_upgrade_error_code": "",
"auth_upgrade_error": "",
}).Error; err != nil {
return err
}
if !QueueLegacyRelationshipUpgrade(nodeID) {
return errors.New("node authentication upgrade queue is full")
}
return nil
}
func classifyAuthUpgradeFailure(err error) authUpgradeFailure {
failure := authUpgradeFailure{
Step: model.NodeAuthUpgradeStepRequest,
Code: model.NodeAuthUpgradeErrorInternal,
Message: "The authentication upgrade failed because of an internal error.",
}
var upgradeErr *relationshipUpgradeError
if errors.As(err, &upgradeErr) {
failure.Step = upgradeErr.Step
}
if errors.Is(err, ErrRelationshipUnsupported) {
failure.Code = model.NodeAuthUpgradeErrorTargetUnsupported
failure.Message = "The target node does not support paired signatures yet."
return failure
}
if errors.Is(err, ErrLegacySecretMissing) {
failure.Code = model.NodeAuthUpgradeErrorMissingLegacySecret
failure.Message = "The stored legacy node secret is unavailable."
return failure
}
if errors.Is(err, ErrUpgradeProofInvalid) {
failure.Code = model.NodeAuthUpgradeErrorInvalidConfirmation
failure.Message = "The target node returned an invalid upgrade confirmation."
return failure
}
var httpErr *relationshipHTTPError
if errors.As(err, &httpErr) {
if httpErr.StatusCode == http.StatusUnauthorized || httpErr.StatusCode == http.StatusForbidden {
failure.Code = model.NodeAuthUpgradeErrorAuthenticationRejected
failure.Message = "The target node rejected the stored node secret."
return failure
}
failure.Code = model.NodeAuthUpgradeErrorTargetRejected
failure.Message = "The target node rejected the authentication upgrade request."
return failure
}
if errors.Is(err, context.DeadlineExceeded) {
failure.Code = model.NodeAuthUpgradeErrorTimeout
failure.Message = "The target node did not respond before the upgrade timed out."
return failure
}
var netErr net.Error
if errors.As(err, &netErr) {
failure.Code = model.NodeAuthUpgradeErrorConnectionFailed
failure.Message = "The target node could not be reached."
return failure
}
var syntaxErr *json.SyntaxError
if errors.As(err, &syntaxErr) || failure.Step == model.NodeAuthUpgradeStepVerify {
failure.Code = model.NodeAuthUpgradeErrorInvalidResponse
failure.Message = "The target node returned an invalid pairing response."
return failure
}
if failure.Step == model.NodeAuthUpgradeStepPersist {
failure.Code = model.NodeAuthUpgradeErrorPersistenceFailed
failure.Message = "The paired credential could not be saved."
}
return failure
}

View File

@@ -159,6 +159,9 @@ func TestLegacyUpgradeUsesPreV250CompatibleAuthentication(t *testing.T) {
require.NoError(t, database.First(&stored, node.ID).Error)
assert.Equal(t, model.NodeAuthMethodPaired, stored.AuthMethod)
assert.Equal(t, model.NodeCredentialStatusActive, stored.CredentialStatus)
assert.Equal(t, model.NodeAuthUpgradeStatusCompleted, stored.AuthUpgradeStatus)
assert.Equal(t, model.NodeAuthUpgradeStepCompleted, stored.AuthUpgradeStep)
assert.NotNil(t, stored.AuthUpgradeCompletedAt)
assert.Empty(t, stored.Token)
assert.NotEmpty(t, stored.EncryptedLegacySecret, "the retained secret is what makes recovery automatic")
@@ -194,6 +197,11 @@ func TestLegacyUpgradeLeavesOlderNodesOnTheSharedSecret(t *testing.T) {
require.NoError(t, database.First(&stored, node.ID).Error)
assert.Equal(t, model.NodeAuthMethodLegacy, stored.AuthMethod)
assert.NotEmpty(t, stored.EncryptedLegacySecret)
assert.Equal(t, model.NodeAuthUpgradeStatusWaitingTarget, stored.AuthUpgradeStatus)
assert.Equal(t, model.NodeAuthUpgradeErrorTargetUnsupported, stored.AuthUpgradeErrorCode)
assert.Equal(t, model.NodeAuthUpgradeStepRequest, stored.AuthUpgradeStep)
assert.EqualValues(t, 1, stored.AuthUpgradeAttemptCount)
assert.NotNil(t, stored.AuthUpgradeNextRetryAt)
}
func TestLegacyUpgradeRejectsUnconfirmedTarget(t *testing.T) {
@@ -223,3 +231,58 @@ func TestLegacyUpgradeRejectsUnconfirmedTarget(t *testing.T) {
require.NoError(t, database.Model(&model.NodeCredential{}).Where("node_id = ?", node.ID).Count(&count).Error)
assert.Zero(t, count)
}
func TestRunLegacyRelationshipUpgradePersistsVerificationFailure(t *testing.T) {
database := setupUpgradeControllerTest(t)
target := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
writer.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(writer).Encode(pairingResponse{
CredentialID: uuid.NewString(),
TargetInstanceID: testTargetInstanceID,
Confirmation: strings.Repeat("A", 43),
}))
}))
t.Cleanup(target.Close)
node := createLegacyNode(t, database, "invalid-confirmation", target.URL)
now := time.Now()
err := RunLegacyRelationshipUpgrade(context.Background(), node.ID, testControllerInstanceID, now)
require.ErrorIs(t, err, ErrUpgradeProofInvalid)
var stored model.Node
require.NoError(t, database.First(&stored, node.ID).Error)
assert.Equal(t, model.NodeAuthMethodLegacy, stored.AuthMethod)
assert.Equal(t, model.NodeAuthUpgradeStatusFailed, stored.AuthUpgradeStatus)
assert.Equal(t, model.NodeAuthUpgradeStepVerify, stored.AuthUpgradeStep)
assert.Equal(t, model.NodeAuthUpgradeErrorInvalidConfirmation, stored.AuthUpgradeErrorCode)
assert.EqualValues(t, 1, stored.AuthUpgradeAttemptCount)
require.NotNil(t, stored.AuthUpgradeNextRetryAt)
assert.WithinDuration(t, now.Add(relationshipUpgradeRetryDelay), *stored.AuthUpgradeNextRetryAt, time.Millisecond)
var count int64
require.NoError(t, database.Model(&model.NodeCredential{}).Where("node_id = ?", node.ID).Count(&count).Error)
assert.Zero(t, count, "a rejected confirmation must not create a credential")
}
func TestRetryLegacyRelationshipUpgradeRequeuesAFailedNode(t *testing.T) {
database := setupUpgradeControllerTest(t)
node := createLegacyNode(t, database, "failed-node", "https://node.example")
require.NoError(t, database.Model(node).Updates(map[string]any{
"auth_upgrade_status": model.NodeAuthUpgradeStatusFailed,
"auth_upgrade_step": model.NodeAuthUpgradeStepVerify,
"auth_upgrade_error_code": model.NodeAuthUpgradeErrorInvalidConfirmation,
"auth_upgrade_error": "sanitized error",
}).Error)
now := time.Now()
require.NoError(t, RetryLegacyRelationshipUpgrade(node.ID, now))
var stored model.Node
require.NoError(t, database.First(&stored, node.ID).Error)
assert.Equal(t, model.NodeAuthUpgradeStatusPending, stored.AuthUpgradeStatus)
assert.Equal(t, model.NodeAuthUpgradeStepQueued, stored.AuthUpgradeStep)
assert.Empty(t, stored.AuthUpgradeErrorCode)
assert.Empty(t, stored.AuthUpgradeError)
require.NotNil(t, stored.AuthUpgradeNextRetryAt)
assert.WithinDuration(t, now, *stored.AuthUpgradeNextRetryAt, time.Millisecond)
}

View File

@@ -14,18 +14,50 @@ const (
NodeCredentialStatusUnpaired = "unpaired"
NodeCredentialStatusRotating = "rotating"
NodeCredentialStatusRevoked = "revoked"
NodeAuthUpgradeStatusPending = "pending"
NodeAuthUpgradeStatusInProgress = "in_progress"
NodeAuthUpgradeStatusWaitingTarget = "waiting_target"
NodeAuthUpgradeStatusFailed = "failed"
NodeAuthUpgradeStatusCompleted = "completed"
NodeAuthUpgradeStatusPaused = "paused"
NodeAuthUpgradeStepQueued = "queued"
NodeAuthUpgradeStepRequest = "request"
NodeAuthUpgradeStepVerify = "verify"
NodeAuthUpgradeStepPersist = "persist"
NodeAuthUpgradeStepCompleted = "completed"
NodeAuthUpgradeErrorTargetUnsupported = "target_unsupported"
NodeAuthUpgradeErrorTimeout = "timeout"
NodeAuthUpgradeErrorConnectionFailed = "connection_failed"
NodeAuthUpgradeErrorAuthenticationRejected = "authentication_rejected"
NodeAuthUpgradeErrorTargetRejected = "target_rejected"
NodeAuthUpgradeErrorInvalidResponse = "invalid_response"
NodeAuthUpgradeErrorInvalidConfirmation = "invalid_confirmation"
NodeAuthUpgradeErrorPersistenceFailed = "persistence_failed"
NodeAuthUpgradeErrorMissingLegacySecret = "missing_legacy_secret"
NodeAuthUpgradeErrorInternal = "internal"
)
type Node struct {
Model
Name string `json:"name"`
URL string `json:"url"`
Token string `json:"-"`
EncryptedLegacySecret []byte `json:"-"`
AuthMethod string `json:"auth_method" gorm:"default:legacy_secret;index"`
CredentialStatus string `json:"credential_status" gorm:"default:unpaired"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
Enabled bool `json:"enabled" gorm:"default:false"`
Name string `json:"name"`
URL string `json:"url"`
Token string `json:"-"`
EncryptedLegacySecret []byte `json:"-"`
AuthMethod string `json:"auth_method" gorm:"default:legacy_secret;index"`
CredentialStatus string `json:"credential_status" gorm:"default:unpaired"`
LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"`
AuthUpgradeStatus string `json:"auth_upgrade_status" gorm:"index"`
AuthUpgradeStep string `json:"auth_upgrade_step"`
AuthUpgradeAttemptCount uint `json:"auth_upgrade_attempt_count"`
AuthUpgradeAttemptedAt *time.Time `json:"auth_upgrade_attempted_at,omitempty"`
AuthUpgradeNextRetryAt *time.Time `json:"auth_upgrade_next_retry_at,omitempty"`
AuthUpgradeCompletedAt *time.Time `json:"auth_upgrade_completed_at,omitempty"`
AuthUpgradeErrorCode string `json:"auth_upgrade_error_code,omitempty"`
AuthUpgradeError string `json:"auth_upgrade_error,omitempty"`
Enabled bool `json:"enabled" gorm:"default:false"`
}
func (n *Node) HasCredential() bool {