diff --git a/api/cluster/node.go b/api/cluster/node.go index a90939db..4b616dee 100644 --- a/api/cluster/node.go +++ b/api/cluster/node.go @@ -31,19 +31,27 @@ type nodeMutationRequest struct { } type nodeResponse struct { - ID uint64 `json:"id"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Name string `json:"name"` - URL string `json:"url"` - Enabled bool `json:"enabled"` - AuthMethod string `json:"auth_method"` - HasCredential bool `json:"has_credential"` - CredentialStatus string `json:"credential_status"` - LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` - ConnectionError string `json:"connection_error,omitempty"` - ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"` - ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"` + ID uint64 `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Name string `json:"name"` + URL string `json:"url"` + Enabled bool `json:"enabled"` + AuthMethod string `json:"auth_method"` + HasCredential bool `json:"has_credential"` + CredentialStatus string `json:"credential_status"` + LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` + AuthUpgradeStatus string `json:"auth_upgrade_status,omitempty"` + AuthUpgradeStep string `json:"auth_upgrade_step,omitempty"` + AuthUpgradeAttemptCount uint `json:"auth_upgrade_attempt_count"` + AuthUpgradeAttemptedAt *time.Time `json:"auth_upgrade_attempted_at,omitempty"` + AuthUpgradeNextRetryAt *time.Time `json:"auth_upgrade_next_retry_at,omitempty"` + AuthUpgradeCompletedAt *time.Time `json:"auth_upgrade_completed_at,omitempty"` + AuthUpgradeErrorCode string `json:"auth_upgrade_error_code,omitempty"` + AuthUpgradeError string `json:"auth_upgrade_error,omitempty"` + ConnectionError string `json:"connection_error,omitempty"` + ConnectionErrorCode analytic.NodeConnectionErrorCode `json:"connection_error_code,omitempty"` + ConnectionErrorAt *time.Time `json:"connection_error_at,omitempty"` // LegacySecret only ever carries the redaction sentinel, which tells the // edit form a secret is stored without putting it in a list response. LegacySecret string `json:"legacy_secret,omitempty"` @@ -53,17 +61,39 @@ type nodeResponse struct { func newNodeResponse(node *model.Node) nodeResponse { analyticNode := analytic.GetNode(node) + authUpgradeStatus := node.AuthUpgradeStatus + authUpgradeErrorCode := node.AuthUpgradeErrorCode + authUpgradeError := node.AuthUpgradeError + if node.AuthMethod == model.NodeAuthMethodLegacy && authUpgradeStatus == "" { + authUpgradeStatus = model.NodeAuthUpgradeStatusPending + } + if node.AuthMethod == model.NodeAuthMethodLegacy && len(node.EncryptedLegacySecret) == 0 { + authUpgradeStatus = model.NodeAuthUpgradeStatusFailed + authUpgradeErrorCode = model.NodeAuthUpgradeErrorMissingLegacySecret + authUpgradeError = "The stored legacy node secret is unavailable." + } + if node.AuthMethod == model.NodeAuthMethodLegacy && !node.Enabled { + authUpgradeStatus = model.NodeAuthUpgradeStatusPaused + } response := nodeResponse{ - ID: node.ID, - CreatedAt: node.CreatedAt, - UpdatedAt: node.UpdatedAt, - Name: node.Name, - URL: node.URL, - Enabled: node.Enabled, - AuthMethod: node.AuthMethod, - HasCredential: node.HasCredential(), - CredentialStatus: node.CredentialStatus, - LastCredentialUseAt: node.LastCredentialUseAt, + ID: node.ID, + CreatedAt: node.CreatedAt, + UpdatedAt: node.UpdatedAt, + Name: node.Name, + URL: node.URL, + Enabled: node.Enabled, + AuthMethod: node.AuthMethod, + HasCredential: node.HasCredential(), + CredentialStatus: node.CredentialStatus, + LastCredentialUseAt: node.LastCredentialUseAt, + AuthUpgradeStatus: authUpgradeStatus, + AuthUpgradeStep: node.AuthUpgradeStep, + AuthUpgradeAttemptCount: node.AuthUpgradeAttemptCount, + AuthUpgradeAttemptedAt: node.AuthUpgradeAttemptedAt, + AuthUpgradeNextRetryAt: node.AuthUpgradeNextRetryAt, + AuthUpgradeCompletedAt: node.AuthUpgradeCompletedAt, + AuthUpgradeErrorCode: authUpgradeErrorCode, + AuthUpgradeError: authUpgradeError, } if len(node.EncryptedLegacySecret) != 0 { response.LegacySecret = settings.RedactedSensitiveValue @@ -113,16 +143,22 @@ func AddNode(c *gin.Context) { legacySecret := mutationLegacySecret(request) authMethod := model.NodeAuthMethodPaired credentialStatus := model.NodeCredentialStatusUnpaired + authUpgradeStatus := "" + authUpgradeStep := "" if legacySecret != "" { authMethod = model.NodeAuthMethodLegacy credentialStatus = model.NodeCredentialStatusActive + authUpgradeStatus = model.NodeAuthUpgradeStatusPending + authUpgradeStep = model.NodeAuthUpgradeStepQueued } node := &model.Node{ - Name: request.Name, - URL: normalizedURL, - Enabled: request.Enabled, - AuthMethod: authMethod, - CredentialStatus: credentialStatus, + Name: request.Name, + URL: normalizedURL, + Enabled: request.Enabled, + AuthMethod: authMethod, + CredentialStatus: credentialStatus, + AuthUpgradeStatus: authUpgradeStatus, + AuthUpgradeStep: authUpgradeStep, } database := model.UseDB() err = database.Transaction(func(tx *gorm.DB) error { @@ -147,6 +183,9 @@ func AddNode(c *gin.Context) { return } refreshNodeState() + if node.Enabled && legacySecret != "" { + nodeauth.QueueLegacyRelationshipUpgrade(node.ID) + } c.JSON(http.StatusCreated, newNodeResponse(node)) } @@ -185,6 +224,12 @@ func EditNode(c *gin.Context) { updates["encrypted_legacy_secret"] = encrypted updates["auth_method"] = model.NodeAuthMethodLegacy updates["credential_status"] = model.NodeCredentialStatusActive + updates["auth_upgrade_status"] = model.NodeAuthUpgradeStatusPending + updates["auth_upgrade_step"] = model.NodeAuthUpgradeStepQueued + updates["auth_upgrade_next_retry_at"] = time.Now() + updates["auth_upgrade_completed_at"] = nil + updates["auth_upgrade_error_code"] = "" + updates["auth_upgrade_error"] = "" if err := tx.Unscoped().Where("node_id = ?", node.ID).Delete(&model.NodeCredential{}).Error; err != nil { return err } @@ -200,9 +245,33 @@ func EditNode(c *gin.Context) { return } refreshNodeState() + if node.Enabled && node.AuthMethod == model.NodeAuthMethodLegacy && len(node.EncryptedLegacySecret) != 0 { + nodeauth.QueueLegacyRelationshipUpgrade(node.ID) + } c.JSON(http.StatusOK, newNodeResponse(node)) } +func RetryNodeAuthUpgrade(c *gin.Context) { + node, ok := findNode(c, false) + if !ok { + return + } + if err := nodeauth.RetryLegacyRelationshipUpgrade(node.ID, time.Now()); err != nil { + status := http.StatusInternalServerError + if errors.Is(err, nodeauth.ErrRelationshipUpgradeAlreadyRunning) || + errors.Is(err, nodeauth.ErrRelationshipUpgradeNotAvailable) { + status = http.StatusConflict + } + c.JSON(status, gin.H{"message": err.Error()}) + return + } + if err := model.UseDB().First(node, node.ID).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"message": err.Error()}) + return + } + c.JSON(http.StatusAccepted, newNodeResponse(node)) +} + func DeleteNode(c *gin.Context) { permanent := cast.ToBool(c.Query("permanent")) node, ok := findNode(c, permanent) diff --git a/api/cluster/node_test.go b/api/cluster/node_test.go index ac95bfe5..b174fd31 100644 --- a/api/cluster/node_test.go +++ b/api/cluster/node_test.go @@ -42,6 +42,7 @@ func TestNodeResponseRedactsAllCredentialMaterial(t *testing.T) { require.NotContains(t, response, "token") require.NotContains(t, response, "private_key") require.Contains(t, response, `"auth_method":"legacy_secret"`) + require.Contains(t, response, `"auth_upgrade_status":"paused"`) require.Contains(t, response, `"has_credential":true`) require.Contains(t, response, `"status":false`) require.NotContains(t, response, `"NodeStat"`) @@ -218,3 +219,16 @@ func TestNodeRouterRegistersRecoveryRoute(t *testing.T) { } t.Fatal("PATCH /api/nodes/:id recovery route is not registered") } + +func TestNodeRouterRegistersAuthenticationUpgradeRetryRoute(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + InitRouter(router.Group("/api")) + + for _, route := range router.Routes() { + if route.Method == http.MethodPost && route.Path == "/api/nodes/:id/auth-upgrade/retry" { + return + } + } + t.Fatal("POST /api/nodes/:id/auth-upgrade/retry is not registered") +} diff --git a/api/cluster/router.go b/api/cluster/router.go index b12975df..313f15a2 100644 --- a/api/cluster/router.go +++ b/api/cluster/router.go @@ -28,6 +28,7 @@ func InitRouter(r *gin.RouterGroup) { admin.GET("nodes/:id/secret", GetNodeSecret) admin.GET("nodes/:id/credentials", GetNodeCredentials) admin.POST("nodes/:id/credentials/rotate", RotateNodeCredential) + admin.POST("nodes/:id/auth-upgrade/retry", RetryNodeAuthUpgrade) admin.GET("node/credentials", ListControllerCredentials) admin.DELETE("node/credentials/:credential_id", RevokeControllerCredential) } diff --git a/app/src/api/node.ts b/app/src/api/node.ts index a35a01d5..be9cb1d8 100644 --- a/app/src/api/node.ts +++ b/app/src/api/node.ts @@ -13,6 +13,14 @@ export interface Node extends ModelBase { has_credential: boolean credential_status: 'active' | 'unpaired' | 'rotating' | 'revoked' last_credential_use_at?: string + auth_upgrade_status?: 'pending' | 'in_progress' | 'waiting_target' | 'failed' | 'completed' | 'paused' + auth_upgrade_step?: 'queued' | 'request' | 'verify' | 'persist' | 'completed' + auth_upgrade_attempt_count: number + auth_upgrade_attempted_at?: string + auth_upgrade_next_retry_at?: string + auth_upgrade_completed_at?: string + auth_upgrade_error_code?: 'target_unsupported' | 'timeout' | 'connection_failed' | 'authentication_rejected' | 'target_rejected' | 'invalid_response' | 'invalid_confirmation' | 'persistence_failed' | 'missing_legacy_secret' | 'internal' + auth_upgrade_error?: string response_at?: string connection_error?: string connection_error_code?: 'clock_skew' @@ -90,6 +98,7 @@ const nodeApi = extendCurdApi(useCurdApi(baseUrl), { syncConfigs, getSecret: (id: number) => http.get<{ value: string }>(`${baseUrl}/${id}/secret`), rotateCredential: (id: number) => http.post(`${baseUrl}/${id}/credentials/rotate`), + retryAuthUpgrade: (id: number) => http.post(`${baseUrl}/${id}/auth-upgrade/retry`), }) export default nodeApi diff --git a/app/src/views/node/NodeAuthStatus.vue b/app/src/views/node/NodeAuthStatus.vue new file mode 100644 index 00000000..a0290f75 --- /dev/null +++ b/app/src/views/node/NodeAuthStatus.vue @@ -0,0 +1,268 @@ + + + + + diff --git a/app/src/views/node/nodeColumns.tsx b/app/src/views/node/nodeColumns.tsx index cec97bbe..6d12ec3e 100644 --- a/app/src/views/node/nodeColumns.tsx +++ b/app/src/views/node/nodeColumns.tsx @@ -7,14 +7,7 @@ import { Badge, InputPassword, Popover, Tag } from 'ant-design-vue' import { h } from 'vue' import nodeApi from '@/api/node' import { SensitiveInput } from '@/components/SensitiveString' - -// Only a credential that is not simply healthy is worth its own word: the -// remaining states collapse into naming the authentication method itself. -const unhealthyCredentialMap: Record string }> = { - rotating: { color: 'blue', text: () => $gettext('Rotating') }, - unpaired: { color: 'default', text: () => $gettext('Unpaired') }, - revoked: { color: 'red', text: () => $gettext('Revoked') }, -} +import NodeAuthStatus from './NodeAuthStatus.vue' function renderConnectionErrorContent(record: Node) { if (record.connection_error_code !== 'clock_skew') { @@ -142,18 +135,9 @@ const columns: StdTableColumn[] = [{ }, { title: () => $gettext('Authentication'), dataIndex: 'auth_method', - customRender: ({ record }: CustomRenderArgs) => { - if (record.auth_method !== 'paired_ed25519') - return {$gettext('Legacy secret')} - - const unhealthy = unhealthyCredentialMap[record.credential_status as string] - if (unhealthy) - return {unhealthy.text()} - - return {$gettext('Paired signature')} - }, + customRender: ({ record }: CustomRenderArgs) => , pure: true, - width: 140, + width: 160, }, { title: () => $gettext('Status'), dataIndex: 'status', diff --git a/e2e/tests/node-auth-upgrade-status.spec.ts b/e2e/tests/node-auth-upgrade-status.spec.ts new file mode 100644 index 00000000..f5bd194a --- /dev/null +++ b/e2e/tests/node-auth-upgrade-status.spec.ts @@ -0,0 +1,131 @@ +import { expect, test } from '@playwright/test' +import { expectTableRows, gotoRoute } from './helpers' + +test('node list explains authentication upgrade progress and failure', async ({ page }, testInfo) => { + await page.setViewportSize({ width: 1600, height: 1000 }) + + await page.route('**/api/self_check', async route => { + await route.fulfill({ status: 200, contentType: 'application/json', json: [] }) + }) + + await page.route('**/api/nodes?**', async route => { + if (route.request().method() !== 'GET') { + await route.continue() + return + } + + const fixture = { + created_at: '2026-08-17T08:00:00Z', + updated_at: '2026-08-17T09:00:00Z', + url: 'https://node.example.test:9000', + version: 'v2.5.9', + status: true, + enabled: true, + auth_method: 'legacy_secret', + credential_status: 'active', + has_credential: true, + auth_upgrade_attempt_count: 0, + } + + const data = [ + { + ...fixture, + id: 9101, + name: 'edge-pending', + status: true, + enabled: true, + auth_method: 'legacy_secret', + credential_status: 'active', + auth_upgrade_status: 'pending', + auth_upgrade_step: 'queued', + auth_upgrade_attempt_count: 0, + }, + { + ...fixture, + id: 9102, + name: 'edge-upgrading', + status: true, + enabled: true, + auth_method: 'legacy_secret', + credential_status: 'active', + auth_upgrade_status: 'in_progress', + auth_upgrade_step: 'verify', + auth_upgrade_attempt_count: 1, + auth_upgrade_attempted_at: '2026-08-17T09:22:10Z', + }, + { + ...fixture, + id: 9103, + name: 'edge-waiting-target', + status: true, + enabled: true, + auth_method: 'legacy_secret', + credential_status: 'active', + auth_upgrade_status: 'waiting_target', + auth_upgrade_step: 'request', + auth_upgrade_attempt_count: 1, + auth_upgrade_attempted_at: '2026-08-17T09:18:32Z', + auth_upgrade_next_retry_at: '2026-08-17T10:18:32Z', + auth_upgrade_error_code: 'target_unsupported', + }, + { + ...fixture, + id: 9104, + name: 'edge-upgrade-failed', + status: true, + enabled: true, + auth_method: 'legacy_secret', + credential_status: 'active', + auth_upgrade_status: 'failed', + auth_upgrade_step: 'verify', + auth_upgrade_attempt_count: 2, + auth_upgrade_attempted_at: '2026-08-17T09:15:06Z', + auth_upgrade_next_retry_at: '2026-08-17T10:15:06Z', + auth_upgrade_error_code: 'invalid_confirmation', + auth_upgrade_error: 'The target node returned an invalid upgrade confirmation.', + }, + ] + await route.fulfill({ + status: 200, + contentType: 'application/json', + json: { + data, + pagination: { + total: data.length, + per_page: 20, + current_page: 1, + total_pages: 1, + }, + }, + }) + }) + + await gotoRoute(page, '/nodes') + const rows = await expectTableRows(page, 4) + await expect(rows.filter({ hasText: 'edge-pending' })).toContainText('Upgrade pending') + await expect(rows.filter({ hasText: 'edge-upgrading' })).toContainText('Upgrading') + await expect(rows.filter({ hasText: 'edge-waiting-target' })).toContainText('Waiting for target') + await expect(rows.filter({ hasText: 'edge-upgrade-failed' })).toContainText('Upgrade failed') + + await page.screenshot({ + path: testInfo.outputPath('node-auth-upgrade-overview.png'), + fullPage: true, + }) + + const failedRow = rows.filter({ hasText: 'edge-upgrade-failed' }) + await failedRow.getByRole('button', { name: 'Authentication upgrade failed' }).click() + + const popover = page.locator('.ant-popover').filter({ hasText: 'Authentication upgrade failed' }) + await expect(popover).toBeVisible() + await expect(popover).toContainText('The target returned an invalid upgrade confirmation.') + await expect(popover).toContainText('Verify target confirmation') + await expect(popover).toContainText('Retry authentication upgrade') + await expect(popover).toContainText('invalid_confirmation') + await popover.getByText('Technical details').click() + await expect(popover.getByText('invalid_confirmation')).toBeVisible() + + await page.screenshot({ + path: testInfo.outputPath('node-auth-upgrade-failed.png'), + fullPage: true, + }) +}) diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 0d860818..c6981a8b 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -4,6 +4,8 @@ import ( "context" "github.com/0xJacky/Nginx-UI/internal/cert" + "github.com/0xJacky/Nginx-UI/internal/nodeauth" + "github.com/0xJacky/Nginx-UI/settings" "github.com/go-co-op/gocron/v2" "github.com/uozi-tech/cosy/logger" ) @@ -55,6 +57,7 @@ func InitCronJobs(ctx context.Context) { } // Initialize automatic node credential upgrade and rotation. + nodeauth.StartRelationshipUpgradeWorker(ctx, settings.NodeSettings.InstanceID) _, err = setupNodeCredentialMaintenanceJob(s) if err != nil { logger.Fatalf("NodeCredentialMaintenance Err: %v\n", err) diff --git a/internal/nodeauth/relationship.go b/internal/nodeauth/relationship.go index 380b1065..3c1ae45d 100644 --- a/internal/nodeauth/relationship.go +++ b/internal/nodeauth/relationship.go @@ -54,6 +54,29 @@ type MaintenanceIssue struct { Err error } +type relationshipUpgradeError struct { + Step string + Err error +} + +func (e *relationshipUpgradeError) Error() string { + return e.Err.Error() +} + +func (e *relationshipUpgradeError) Unwrap() error { + return e.Err +} + +type relationshipHTTPError struct { + StatusCode int + Status string + Message string +} + +func (e *relationshipHTTPError) Error() string { + return fmt.Sprintf("node returned %s: %s", e.Status, e.Message) +} + type upgradeRequest struct { ControllerInstanceID string `json:"controller_instance_id"` PublicKey string `json:"public_key"` @@ -77,40 +100,62 @@ type rotationRequest struct { // before the new credential is stored, so a substituted response cannot leave // the relationship relying on a key the real target never issued. func UpgradeLegacyRelationship(ctx context.Context, node *model.Node, controllerInstanceID string) (*PairingResult, error) { + return upgradeLegacyRelationship(ctx, node, controllerInstanceID, nil) +} + +func upgradeLegacyRelationship(ctx context.Context, node *model.Node, controllerInstanceID string, + reportStep func(string), +) (*PairingResult, error) { if node == nil { - return nil, errors.New("node is required") + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: errors.New("node is required")} } if len(node.EncryptedLegacySecret) == 0 { - return nil, ErrLegacySecretMissing + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: ErrLegacySecretMissing} } if _, err := uuid.Parse(controllerInstanceID); err != nil { - return nil, errors.New("invalid controller instance ID") + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: errors.New("invalid controller instance ID")} } secret, err := DecryptPrivateCredential(LegacyCredentialPurpose(node.ID), node.EncryptedLegacySecret) if err != nil { - return nil, err + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err} } publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { - return nil, err + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err} } + reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepRequest) var response pairingResponse if err := sendRelationshipJSON(ctx, node, http.MethodPost, "/api/node/pair/upgrade", upgradeRequest{ ControllerInstanceID: controllerInstanceID, PublicKey: base64.RawURLEncoding.EncodeToString(publicKey), }, &response, true); err != nil { - return nil, err + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepRequest, Err: err} } + reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepVerify) if err := validatePairingResponse(&response); err != nil { - return nil, err + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepVerify, Err: err} } if err := VerifyUpgradeConfirmation(secret, controllerInstanceID, publicKey, response.CredentialID, response.TargetInstanceID, response.Confirmation); err != nil { - return nil, fmt.Errorf("target node did not confirm the upgrade: %w", err) + return nil, &relationshipUpgradeError{ + Step: model.NodeAuthUpgradeStepVerify, + Err: fmt.Errorf("target node did not confirm the upgrade: %w", err), + } + } + reportRelationshipUpgradeStep(reportStep, model.NodeAuthUpgradeStepPersist) + result, err := persistRelationship(node, &response, publicKey, privateKey) + if err != nil { + return nil, &relationshipUpgradeError{Step: model.NodeAuthUpgradeStepPersist, Err: err} + } + return result, nil +} + +func reportRelationshipUpgradeStep(reportStep func(string), step string) { + if reportStep != nil { + reportStep(step) } - return persistRelationship(node, &response, publicKey, privateKey) } func validatePairingResponse(response *pairingResponse) error { @@ -157,10 +202,17 @@ func persistRelationship(node *model.Node, response *pairingResponse, // node that loses its controller credential (a restored backup, a // revoked relationship) be re-upgraded automatically instead of waiting // for someone to notice and pair it by hand. + now := time.Now() return tx.Model(&model.Node{}).Where("id = ?", node.ID).Updates(map[string]any{ - "token": "", - "auth_method": model.NodeAuthMethodPaired, - "credential_status": model.NodeCredentialStatusActive, + "token": "", + "auth_method": model.NodeAuthMethodPaired, + "credential_status": model.NodeCredentialStatusActive, + "auth_upgrade_status": model.NodeAuthUpgradeStatusCompleted, + "auth_upgrade_step": model.NodeAuthUpgradeStepCompleted, + "auth_upgrade_completed_at": now, + "auth_upgrade_next_retry_at": nil, + "auth_upgrade_error_code": "", + "auth_upgrade_error": "", }).Error }) if err != nil { @@ -281,11 +333,14 @@ func MaintainRelationships(ctx context.Context, controllerInstanceID string, now if len(node.EncryptedLegacySecret) == 0 { continue } + if node.AuthUpgradeNextRetryAt != nil && node.AuthUpgradeNextRetryAt.After(now) { + continue + } // A node that has not been upgraded yet simply keeps using the shared // secret. That is an expected state during a rolling upgrade, not a // fault worth reporting on every pass. - if _, err := UpgradeLegacyRelationship(ctx, node, controllerInstanceID); err != nil && - !errors.Is(err, ErrRelationshipUnsupported) { + if err := RunLegacyRelationshipUpgrade(ctx, node.ID, controllerInstanceID, now); err != nil && + !errors.Is(err, ErrRelationshipUpgradeAlreadyRunning) { issues = append(issues, MaintenanceIssue{NodeID: node.ID, Operation: "upgrade", Err: err}) } case model.NodeAuthMethodPaired: @@ -366,7 +421,11 @@ func sendRelationshipJSON(ctx context.Context, node *model.Node, method, path st if message == "" { message = http.StatusText(httpResponse.StatusCode) } - return fmt.Errorf("node returned %s: %s", httpResponse.Status, message) + return &relationshipHTTPError{ + StatusCode: httpResponse.StatusCode, + Status: httpResponse.Status, + Message: message, + } } if response != nil && len(responseBody) != 0 { if err := json.Unmarshal(responseBody, response); err != nil { diff --git a/internal/nodeauth/upgrade_state.go b/internal/nodeauth/upgrade_state.go new file mode 100644 index 00000000..72cb997f --- /dev/null +++ b/internal/nodeauth/upgrade_state.go @@ -0,0 +1,225 @@ +package nodeauth + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "sync" + "time" + + "github.com/0xJacky/Nginx-UI/model" + "github.com/uozi-tech/cosy/logger" + "gorm.io/gorm" +) + +const ( + relationshipUpgradeRetryDelay = time.Hour + relationshipUpgradeStaleAfter = 10 * time.Minute + relationshipUpgradeQueueSize = 128 +) + +var ( + ErrRelationshipUpgradeAlreadyRunning = errors.New("node authentication upgrade is already running") + ErrRelationshipUpgradeNotAvailable = errors.New("node authentication upgrade is not available") + + relationshipUpgradeQueue = make(chan uint64, relationshipUpgradeQueueSize) + relationshipUpgradeWorkerOnce sync.Once +) + +type authUpgradeFailure struct { + Step string + Code string + Message string +} + +func StartRelationshipUpgradeWorker(ctx context.Context, controllerInstanceID string) { + relationshipUpgradeWorkerOnce.Do(func() { + go func() { + for { + select { + case <-ctx.Done(): + return + case nodeID := <-relationshipUpgradeQueue: + attemptCtx, cancel := context.WithTimeout(ctx, time.Minute) + err := RunLegacyRelationshipUpgrade(attemptCtx, nodeID, controllerInstanceID, time.Now()) + cancel() + if err != nil && !errors.Is(err, ErrRelationshipUpgradeAlreadyRunning) { + logger.Warnf("Automatic node authentication upgrade failed for node %d: %v", nodeID, err) + } + } + } + }() + }) +} + +func QueueLegacyRelationshipUpgrade(nodeID uint64) bool { + select { + case relationshipUpgradeQueue <- nodeID: + return true + default: + return false + } +} + +func RunLegacyRelationshipUpgrade(ctx context.Context, nodeID uint64, controllerInstanceID string, + now time.Time, +) error { + database := model.UseDB() + if database == nil { + return errors.New("node authentication database is unavailable") + } + + var node model.Node + if err := database.First(&node, nodeID).Error; err != nil { + return err + } + if node.AuthMethod != model.NodeAuthMethodLegacy || !node.Enabled || len(node.EncryptedLegacySecret) == 0 { + return ErrRelationshipUpgradeNotAvailable + } + + staleBefore := now.Add(-relationshipUpgradeStaleAfter) + result := database.Model(&model.Node{}). + Where("id = ? AND auth_method = ?", nodeID, model.NodeAuthMethodLegacy). + Where("auth_upgrade_status IS NULL OR auth_upgrade_status <> ? OR auth_upgrade_attempted_at IS NULL OR auth_upgrade_attempted_at < ?", + model.NodeAuthUpgradeStatusInProgress, staleBefore). + Updates(map[string]any{ + "auth_upgrade_status": model.NodeAuthUpgradeStatusInProgress, + "auth_upgrade_step": model.NodeAuthUpgradeStepRequest, + "auth_upgrade_attempt_count": gorm.Expr("auth_upgrade_attempt_count + 1"), + "auth_upgrade_attempted_at": now, + "auth_upgrade_next_retry_at": nil, + "auth_upgrade_error_code": "", + "auth_upgrade_error": "", + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrRelationshipUpgradeAlreadyRunning + } + + reportStep := func(step string) { + _ = database.Model(&model.Node{}). + Where("id = ? AND auth_upgrade_status = ?", nodeID, model.NodeAuthUpgradeStatusInProgress). + Update("auth_upgrade_step", step).Error + } + _, err := upgradeLegacyRelationship(ctx, &node, controllerInstanceID, reportStep) + if err == nil { + return nil + } + + failure := classifyAuthUpgradeFailure(err) + nextRetryAt := now.Add(relationshipUpgradeRetryDelay) + status := model.NodeAuthUpgradeStatusFailed + if failure.Code == model.NodeAuthUpgradeErrorTargetUnsupported { + status = model.NodeAuthUpgradeStatusWaitingTarget + } + if updateErr := database.Model(&model.Node{}).Where("id = ?", nodeID).Updates(map[string]any{ + "auth_upgrade_status": status, + "auth_upgrade_step": failure.Step, + "auth_upgrade_next_retry_at": nextRetryAt, + "auth_upgrade_error_code": failure.Code, + "auth_upgrade_error": failure.Message, + }).Error; updateErr != nil { + return errors.Join(err, updateErr) + } + if status == model.NodeAuthUpgradeStatusWaitingTarget { + return nil + } + return err +} + +func RetryLegacyRelationshipUpgrade(nodeID uint64, now time.Time) error { + database := model.UseDB() + if database == nil { + return errors.New("node authentication database is unavailable") + } + + var node model.Node + if err := database.First(&node, nodeID).Error; err != nil { + return err + } + if node.AuthMethod != model.NodeAuthMethodLegacy || !node.Enabled || len(node.EncryptedLegacySecret) == 0 { + return ErrRelationshipUpgradeNotAvailable + } + if node.AuthUpgradeStatus == model.NodeAuthUpgradeStatusInProgress && node.AuthUpgradeAttemptedAt != nil && + node.AuthUpgradeAttemptedAt.After(now.Add(-relationshipUpgradeStaleAfter)) { + return ErrRelationshipUpgradeAlreadyRunning + } + + if err := database.Model(&node).Updates(map[string]any{ + "auth_upgrade_status": model.NodeAuthUpgradeStatusPending, + "auth_upgrade_step": model.NodeAuthUpgradeStepQueued, + "auth_upgrade_next_retry_at": now, + "auth_upgrade_error_code": "", + "auth_upgrade_error": "", + }).Error; err != nil { + return err + } + if !QueueLegacyRelationshipUpgrade(nodeID) { + return errors.New("node authentication upgrade queue is full") + } + return nil +} + +func classifyAuthUpgradeFailure(err error) authUpgradeFailure { + failure := authUpgradeFailure{ + Step: model.NodeAuthUpgradeStepRequest, + Code: model.NodeAuthUpgradeErrorInternal, + Message: "The authentication upgrade failed because of an internal error.", + } + var upgradeErr *relationshipUpgradeError + if errors.As(err, &upgradeErr) { + failure.Step = upgradeErr.Step + } + if errors.Is(err, ErrRelationshipUnsupported) { + failure.Code = model.NodeAuthUpgradeErrorTargetUnsupported + failure.Message = "The target node does not support paired signatures yet." + return failure + } + if errors.Is(err, ErrLegacySecretMissing) { + failure.Code = model.NodeAuthUpgradeErrorMissingLegacySecret + failure.Message = "The stored legacy node secret is unavailable." + return failure + } + if errors.Is(err, ErrUpgradeProofInvalid) { + failure.Code = model.NodeAuthUpgradeErrorInvalidConfirmation + failure.Message = "The target node returned an invalid upgrade confirmation." + return failure + } + var httpErr *relationshipHTTPError + if errors.As(err, &httpErr) { + if httpErr.StatusCode == http.StatusUnauthorized || httpErr.StatusCode == http.StatusForbidden { + failure.Code = model.NodeAuthUpgradeErrorAuthenticationRejected + failure.Message = "The target node rejected the stored node secret." + return failure + } + failure.Code = model.NodeAuthUpgradeErrorTargetRejected + failure.Message = "The target node rejected the authentication upgrade request." + return failure + } + if errors.Is(err, context.DeadlineExceeded) { + failure.Code = model.NodeAuthUpgradeErrorTimeout + failure.Message = "The target node did not respond before the upgrade timed out." + return failure + } + var netErr net.Error + if errors.As(err, &netErr) { + failure.Code = model.NodeAuthUpgradeErrorConnectionFailed + failure.Message = "The target node could not be reached." + return failure + } + var syntaxErr *json.SyntaxError + if errors.As(err, &syntaxErr) || failure.Step == model.NodeAuthUpgradeStepVerify { + failure.Code = model.NodeAuthUpgradeErrorInvalidResponse + failure.Message = "The target node returned an invalid pairing response." + return failure + } + if failure.Step == model.NodeAuthUpgradeStepPersist { + failure.Code = model.NodeAuthUpgradeErrorPersistenceFailed + failure.Message = "The paired credential could not be saved." + } + return failure +} diff --git a/internal/nodeauth/upgrade_test.go b/internal/nodeauth/upgrade_test.go index 64c46603..185f8524 100644 --- a/internal/nodeauth/upgrade_test.go +++ b/internal/nodeauth/upgrade_test.go @@ -159,6 +159,9 @@ func TestLegacyUpgradeUsesPreV250CompatibleAuthentication(t *testing.T) { require.NoError(t, database.First(&stored, node.ID).Error) assert.Equal(t, model.NodeAuthMethodPaired, stored.AuthMethod) assert.Equal(t, model.NodeCredentialStatusActive, stored.CredentialStatus) + assert.Equal(t, model.NodeAuthUpgradeStatusCompleted, stored.AuthUpgradeStatus) + assert.Equal(t, model.NodeAuthUpgradeStepCompleted, stored.AuthUpgradeStep) + assert.NotNil(t, stored.AuthUpgradeCompletedAt) assert.Empty(t, stored.Token) assert.NotEmpty(t, stored.EncryptedLegacySecret, "the retained secret is what makes recovery automatic") @@ -194,6 +197,11 @@ func TestLegacyUpgradeLeavesOlderNodesOnTheSharedSecret(t *testing.T) { require.NoError(t, database.First(&stored, node.ID).Error) assert.Equal(t, model.NodeAuthMethodLegacy, stored.AuthMethod) assert.NotEmpty(t, stored.EncryptedLegacySecret) + assert.Equal(t, model.NodeAuthUpgradeStatusWaitingTarget, stored.AuthUpgradeStatus) + assert.Equal(t, model.NodeAuthUpgradeErrorTargetUnsupported, stored.AuthUpgradeErrorCode) + assert.Equal(t, model.NodeAuthUpgradeStepRequest, stored.AuthUpgradeStep) + assert.EqualValues(t, 1, stored.AuthUpgradeAttemptCount) + assert.NotNil(t, stored.AuthUpgradeNextRetryAt) } func TestLegacyUpgradeRejectsUnconfirmedTarget(t *testing.T) { @@ -223,3 +231,58 @@ func TestLegacyUpgradeRejectsUnconfirmedTarget(t *testing.T) { require.NoError(t, database.Model(&model.NodeCredential{}).Where("node_id = ?", node.ID).Count(&count).Error) assert.Zero(t, count) } + +func TestRunLegacyRelationshipUpgradePersistsVerificationFailure(t *testing.T) { + database := setupUpgradeControllerTest(t) + target := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(writer).Encode(pairingResponse{ + CredentialID: uuid.NewString(), + TargetInstanceID: testTargetInstanceID, + Confirmation: strings.Repeat("A", 43), + })) + })) + t.Cleanup(target.Close) + + node := createLegacyNode(t, database, "invalid-confirmation", target.URL) + now := time.Now() + err := RunLegacyRelationshipUpgrade(context.Background(), node.ID, testControllerInstanceID, now) + require.ErrorIs(t, err, ErrUpgradeProofInvalid) + + var stored model.Node + require.NoError(t, database.First(&stored, node.ID).Error) + assert.Equal(t, model.NodeAuthMethodLegacy, stored.AuthMethod) + assert.Equal(t, model.NodeAuthUpgradeStatusFailed, stored.AuthUpgradeStatus) + assert.Equal(t, model.NodeAuthUpgradeStepVerify, stored.AuthUpgradeStep) + assert.Equal(t, model.NodeAuthUpgradeErrorInvalidConfirmation, stored.AuthUpgradeErrorCode) + assert.EqualValues(t, 1, stored.AuthUpgradeAttemptCount) + require.NotNil(t, stored.AuthUpgradeNextRetryAt) + assert.WithinDuration(t, now.Add(relationshipUpgradeRetryDelay), *stored.AuthUpgradeNextRetryAt, time.Millisecond) + + var count int64 + require.NoError(t, database.Model(&model.NodeCredential{}).Where("node_id = ?", node.ID).Count(&count).Error) + assert.Zero(t, count, "a rejected confirmation must not create a credential") +} + +func TestRetryLegacyRelationshipUpgradeRequeuesAFailedNode(t *testing.T) { + database := setupUpgradeControllerTest(t) + node := createLegacyNode(t, database, "failed-node", "https://node.example") + require.NoError(t, database.Model(node).Updates(map[string]any{ + "auth_upgrade_status": model.NodeAuthUpgradeStatusFailed, + "auth_upgrade_step": model.NodeAuthUpgradeStepVerify, + "auth_upgrade_error_code": model.NodeAuthUpgradeErrorInvalidConfirmation, + "auth_upgrade_error": "sanitized error", + }).Error) + + now := time.Now() + require.NoError(t, RetryLegacyRelationshipUpgrade(node.ID, now)) + + var stored model.Node + require.NoError(t, database.First(&stored, node.ID).Error) + assert.Equal(t, model.NodeAuthUpgradeStatusPending, stored.AuthUpgradeStatus) + assert.Equal(t, model.NodeAuthUpgradeStepQueued, stored.AuthUpgradeStep) + assert.Empty(t, stored.AuthUpgradeErrorCode) + assert.Empty(t, stored.AuthUpgradeError) + require.NotNil(t, stored.AuthUpgradeNextRetryAt) + assert.WithinDuration(t, now, *stored.AuthUpgradeNextRetryAt, time.Millisecond) +} diff --git a/model/node.go b/model/node.go index 98319a03..d3949f80 100644 --- a/model/node.go +++ b/model/node.go @@ -14,18 +14,50 @@ const ( NodeCredentialStatusUnpaired = "unpaired" NodeCredentialStatusRotating = "rotating" NodeCredentialStatusRevoked = "revoked" + + NodeAuthUpgradeStatusPending = "pending" + NodeAuthUpgradeStatusInProgress = "in_progress" + NodeAuthUpgradeStatusWaitingTarget = "waiting_target" + NodeAuthUpgradeStatusFailed = "failed" + NodeAuthUpgradeStatusCompleted = "completed" + NodeAuthUpgradeStatusPaused = "paused" + + NodeAuthUpgradeStepQueued = "queued" + NodeAuthUpgradeStepRequest = "request" + NodeAuthUpgradeStepVerify = "verify" + NodeAuthUpgradeStepPersist = "persist" + NodeAuthUpgradeStepCompleted = "completed" + + NodeAuthUpgradeErrorTargetUnsupported = "target_unsupported" + NodeAuthUpgradeErrorTimeout = "timeout" + NodeAuthUpgradeErrorConnectionFailed = "connection_failed" + NodeAuthUpgradeErrorAuthenticationRejected = "authentication_rejected" + NodeAuthUpgradeErrorTargetRejected = "target_rejected" + NodeAuthUpgradeErrorInvalidResponse = "invalid_response" + NodeAuthUpgradeErrorInvalidConfirmation = "invalid_confirmation" + NodeAuthUpgradeErrorPersistenceFailed = "persistence_failed" + NodeAuthUpgradeErrorMissingLegacySecret = "missing_legacy_secret" + NodeAuthUpgradeErrorInternal = "internal" ) type Node struct { Model - Name string `json:"name"` - URL string `json:"url"` - Token string `json:"-"` - EncryptedLegacySecret []byte `json:"-"` - AuthMethod string `json:"auth_method" gorm:"default:legacy_secret;index"` - CredentialStatus string `json:"credential_status" gorm:"default:unpaired"` - LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` - Enabled bool `json:"enabled" gorm:"default:false"` + Name string `json:"name"` + URL string `json:"url"` + Token string `json:"-"` + EncryptedLegacySecret []byte `json:"-"` + AuthMethod string `json:"auth_method" gorm:"default:legacy_secret;index"` + CredentialStatus string `json:"credential_status" gorm:"default:unpaired"` + LastCredentialUseAt *time.Time `json:"last_credential_use_at,omitempty"` + AuthUpgradeStatus string `json:"auth_upgrade_status" gorm:"index"` + AuthUpgradeStep string `json:"auth_upgrade_step"` + AuthUpgradeAttemptCount uint `json:"auth_upgrade_attempt_count"` + AuthUpgradeAttemptedAt *time.Time `json:"auth_upgrade_attempted_at,omitempty"` + AuthUpgradeNextRetryAt *time.Time `json:"auth_upgrade_next_retry_at,omitempty"` + AuthUpgradeCompletedAt *time.Time `json:"auth_upgrade_completed_at,omitempty"` + AuthUpgradeErrorCode string `json:"auth_upgrade_error_code,omitempty"` + AuthUpgradeError string `json:"auth_upgrade_error,omitempty"` + Enabled bool `json:"enabled" gorm:"default:false"` } func (n *Node) HasCredential() bool {