diff --git a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go index 4d4de75991..97eab818df 100644 --- a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go +++ b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go @@ -72,16 +72,22 @@ func (p *DiskSchedtagPredicate) GetResources(c core.Candidater) []ISchedtagCandi return ret } -func (p *DiskSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) error { +func (p *DiskSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) core.PredicateFailureReason { storage := res.(*api.CandidateStorage) if storage.Status == computeapi.STORAGE_OFFLINE || storage.Enabled.IsFalse() { - return fmt.Errorf("Storage status is %s, enable is %v", storage.Status, storage.Enabled) + return &FailReason{ + fmt.Sprintf("Storage status is %s, enable is %v", storage.Status, storage.Enabled), + StorageEnable, + } } d := input.(*diskW) if d.Storage != "" { if storage.Id != d.Storage && storage.Name != d.Storage { - return fmt.Errorf("Storage name %s != (%s:%s)", d.Storage, storage.Name, storage.Id) + return &FailReason{ + fmt.Sprintf("Storage name %s != (%s:%s)", d.Storage, storage.Name, storage.Id), + StorageMatch, + } } } if c.Getter().ResourceType() == computeapi.HostResourceTypePrepaidRecycle { @@ -89,13 +95,19 @@ func (p *DiskSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candidat } if !(len(d.Backend) == 0 || d.Backend == computeapi.STORAGE_LOCAL) { if storage.StorageType != d.Backend { - return fmt.Errorf("Storage %s backend %s != %s", storage.Name, storage.StorageType, d.Backend) + return &FailReason{ + fmt.Sprintf("Storage %s backend %s != %s", storage.Name, storage.StorageType, d.Backend), + StorageType, + } } } storageTypes := p.GetHypervisorDriver().GetStorageTypes() if len(storageTypes) != 0 && !utils.IsInStringArray(storage.StorageType, storageTypes) { - return fmt.Errorf("Storage %s storage type %s not in %v", storage.Name, storage.StorageType, storageTypes) + return &FailReason{ + fmt.Sprintf("Storage %s storage type %s not in %v", storage.Name, storage.StorageType, storageTypes), + StorageType, + } } return nil } diff --git a/pkg/scheduler/algorithm/predicates/error.go b/pkg/scheduler/algorithm/predicates/error.go index 35e882b008..26a3151ccb 100644 --- a/pkg/scheduler/algorithm/predicates/error.go +++ b/pkg/scheduler/algorithm/predicates/error.go @@ -94,3 +94,32 @@ func (ure *UnexceptedResourceError) Error() string { func (ure *UnexceptedResourceError) GetReason() string { return ure.Error() } + +type FailReason struct { + Reason string + Type string +} + +func (r FailReason) GetReason() string { + return r.Reason +} + +func (r FailReason) GetType() string { + return r.Type +} + +const ( + NetworkPrivate = "network_private" + NetworkPublic = "network_public" + NetworkTypeMatch = "network_type" + NetworkMatch = "network_match" + NetworkWire = "network_wire" + NetworkOwner = "network_owner" + NetworkDomain = "network_domain" + NetworkRange = "network_range" + NetworkFreeCount = "network_free_count" + + StorageEnable = "storage_status" + StorageMatch = "storage_match" + StorageType = "storage_type" +) diff --git a/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go index a2143f9d52..4119d55d9f 100644 --- a/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go +++ b/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go @@ -75,42 +75,63 @@ func (p *NetworkSchedtagPredicate) GetResources(c core.Candidater) []ISchedtagCa return ret } -func (p *NetworkSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) error { +func (p *NetworkSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) core.PredicateFailureReason { network := res.(*api.CandidateNetwork) net := input.(*netW) if net.Network != "" { if network.Id != net.Network && network.Name != net.Network { - return fmt.Errorf("Network name %s != (%s:%s)", net.Network, network.Name, network.Id) + return &FailReason{ + Reason: fmt.Sprintf("Network name %s != (%s:%s)", net.Network, network.Name, network.Id), + Type: NetworkMatch, + } } } if net.Wire != "" { if network.WireId != net.Wire { - return fmt.Errorf("Wire %s != %s", net.Wire, network.WireId) + return &FailReason{ + Reason: fmt.Sprintf("Wire %s != %s", net.Wire, network.WireId), + Type: NetworkWire, + } } } if net.Network == "" { netTypes := p.GetNetworkTypes(net.NetType) if !utils.IsInStringArray(network.ServerType, netTypes) { - return fmt.Errorf("Network %s type %s not in %v", network.Name, network.ServerType, netTypes) + return &FailReason{ + Reason: fmt.Sprintf("Network %s type %s not in %v", network.Name, network.ServerType, netTypes), + Type: NetworkTypeMatch, + } } schedData := u.SchedData() if net.Private { if network.IsPublic { - return fmt.Errorf("Network %s is public", network.Name) + return &FailReason{ + Reason: fmt.Sprintf("Network %s is public", network.Name), + Type: NetworkPublic, + } } if network.ProjectId != schedData.Project && !utils.IsInStringArray(schedData.Project, network.GetSharedProjects()) { - return fmt.Errorf("Network project %s + %v not owner by %s", network.ProjectId, network.GetSharedProjects(), schedData.Project) + return &FailReason{ + Reason: fmt.Sprintf("Network project %s + %v not owner by %s", network.ProjectId, network.GetSharedProjects(), schedData.Project), + Type: NetworkOwner, + } } } else { if !network.IsPublic { - return fmt.Errorf("Network %s is private", network.Name) + return &FailReason{ + fmt.Sprintf("Network %s is private", network.Name), + NetworkPrivate, + } } if rbacutils.TRbacScope(network.PublicScope) == rbacutils.ScopeDomain { netDomain := network.DomainId reqDomain := net.Domain if netDomain != reqDomain { - return fmt.Errorf("Network domain scope %s not owner by %s", netDomain, reqDomain) + return &FailReason{ + fmt.Sprintf("Network domain scope %s not owner by %s", netDomain, reqDomain), + NetworkDomain, + } } } } @@ -119,19 +140,31 @@ func (p *NetworkSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candi if len(net.Address) > 0 { ipAddr, err := netutils.NewIPV4Addr(net.Address) if err != nil { - return fmt.Errorf("Invalid ip address %s: %v", net.Address, err) + return &FailReason{ + fmt.Sprintf("Invalid ip address %s: %v", net.Address, err), + NetworkRange, + } } if !network.GetIPRange().Contains(ipAddr) { - return fmt.Errorf("Address %s not in range", net.Address) + return &FailReason{ + fmt.Sprintf("Address %s not in range", net.Address), + NetworkRange, + } } } free, err := network.GetFreeAddressCount() if err != nil { - return err + return &FailReason{ + Reason: fmt.Sprintf("get free address count: %v", err), + Type: NetworkFreeCount, + } } req := u.SchedData().Count if free < req { - return fmt.Errorf("Network %s no free IPs, free %d, require %d", network.Name, free, req) + return &FailReason{ + Reason: fmt.Sprintf("Network %s no free IPs, free %d, require %d", network.Name, free, req), + Type: NetworkFreeCount, + } } h := NewPredicateHelper(p, u, c) h.SetCapacity(int64(free)) diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 2fd31d9b55..ffd01130f1 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -1,5 +1,3 @@ -// Copyright 2019 Yunion -// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -103,13 +101,34 @@ func (h *PredicateHelper) AppendPredicateFail(reason core.PredicateFailureReason h.predicateFails = append(h.predicateFails, reason) } +type predicateFailure struct { + err core.PredicateFailureError + eType string +} + +func (f predicateFailure) GetReason() string { + return f.err.GetReason() +} + +func (f predicateFailure) GetType() string { + return f.eType +} + func (h *PredicateHelper) AppendPredicateFailMsg(reason string) { - h.AppendPredicateFail(NewUnexceptedResourceError(reason)) + h.AppendPredicateFailMsgWithType(reason, h.predicate.Name()) +} + +func (h *PredicateHelper) AppendPredicateFailMsgWithType(reason string, eType string) { + err := NewUnexceptedResourceError(reason) + h.AppendPredicateFail(&predicateFailure{err: err, eType: eType}) } func (h *PredicateHelper) AppendInsufficientResourceError(req, total, free int64) { h.AppendPredicateFail( - NewInsufficientResourceError(h.Candidate.Getter().Name(), req, total, free)) + &predicateFailure{ + err: NewInsufficientResourceError(h.Candidate.Getter().Name(), req, total, free), + eType: h.predicate.Name(), + }) } // SetCapacity returns the current resource capacity calculated by a filter. @@ -137,6 +156,13 @@ func (h *PredicateHelper) Exclude(reason string) { h.AppendPredicateFailMsg(reason) } +func (h *PredicateHelper) ExcludeByErrors(errs []core.PredicateFailureReason) { + h.SetCapacity(0) + for _, err := range errs { + h.AppendPredicateFail(err) + } +} + func (h *PredicateHelper) Exclude2(predicateName string, current, expected interface{}) { h.Exclude(fmt.Sprintf("%s is '%v', expected '%v'", predicateName, current, expected)) } @@ -204,7 +230,7 @@ type ISchedtagPredicateInstance interface { GetInputs(u *core.Unit) []ISchedtagCustomer GetResources(c core.Candidater) []ISchedtagCandidateResource - IsResourceFitInput(unit *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) error + IsResourceFitInput(unit *core.Unit, c core.Candidater, res ISchedtagCandidateResource, input ISchedtagCustomer) core.PredicateFailureReason DoSelect(c core.Candidater, input ISchedtagCustomer, res []ISchedtagCandidateResource) []ISchedtagCandidateResource AddSelectResult(index int, selectRes []ISchedtagCandidateResource, output *core.AllocatedResource) @@ -354,10 +380,10 @@ func (p *BaseSchedtagPredicate) Execute( h := NewPredicateHelper(sp, u, c) inputRes := p.GetInputResourcesMap(c.IndexKey()) - filterErrs := make([]error, 0) + filterErrs := make([]core.PredicateFailureReason, 0) for idx, input := range inputs { fitResources := make([]ISchedtagCandidateResource, 0) - errs := make([]error, 0) + errs := make([]core.PredicateFailureReason, 0) for _, res := range resources { if err := sp.IsResourceFitInput(u, c, res, input); err == nil { fitResources = append(fitResources, res) @@ -366,20 +392,19 @@ func (p *BaseSchedtagPredicate) Execute( } } if len(fitResources) == 0 { - h.Exclude(fmt.Sprintf("Not found available resources for %s %s: %s", input.Keyword(), input.JSON(input), errors.NewAggregate(errs))) + h.ExcludeByErrors(errs) break } if len(errs) > 0 { - filterErrs = append(filterErrs, errors.NewAggregate(errs)) + filterErrs = append(filterErrs, errs...) } matchedResources, err := p.checkResources(input, fitResources, u, c) if err != nil { - aggErr := errors.NewAggregate(filterErrs) - errMsg := fmt.Sprintf("schedtag: %v", err.Error()) - if aggErr != nil { - errMsg = fmt.Sprintf("%s; filter: %v", errMsg, aggErr.Error()) + if len(filterErrs) > 0 { + h.ExcludeByErrors(filterErrs) } + errMsg := fmt.Sprintf("schedtag: %v", err.Error()) h.Exclude(errMsg) } inputRes[idx] = matchedResources diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index ffd0e5fc1b..13f63679bd 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -221,19 +221,38 @@ type SchedContextDataItem struct { Data map[string]interface{} } +type LogMessage struct { + Type string + Info string +} + +type LogMessages []*LogMessage + +func (ms LogMessages) String() string { + ss := make([]string, 0) + for _, s := range ms { + ss = append(ss, fmt.Sprintf("%s: %s", s.Type, s.Info)) + } + return strings.Join(ss, ",") +} + type SchedLog struct { Candidate string Action string - Message string + Messages LogMessages IsFailed bool } -func NewSchedLog(candidate, action, message string, isFailed bool) SchedLog { - return SchedLog{candidate, action, message, isFailed} +func NewSchedLog(candidate, action string, messages LogMessages, isFailed bool) SchedLog { + return SchedLog{candidate, action, messages, isFailed} } func (log *SchedLog) String() string { - return fmt.Sprintf("%v [%v] %v", log.Candidate, log.Action, log.Message) + prefix := "Success" + if log.IsFailed { + prefix = "Failed" + } + return fmt.Sprintf("%s: %v [%v] %v", prefix, log.Candidate, log.Action, log.Messages.String()) } type SchedLogList []SchedLog @@ -257,7 +276,7 @@ func (logList SchedLogList) Less(i, j int) bool { return r < 0 } - r = strings.Compare(logList[i].Message, logList[j].Message) + r = strings.Compare(logList[i].Messages.String(), logList[j].Messages.String()) if r != 0 { return r < 0 } @@ -282,12 +301,12 @@ func NewSchedLogManager() *SchedLogManager { } } -func (m *SchedLogManager) Append(candidate, action, message string, isFailed bool) { +/*func (m *SchedLogManager) Append(candidate, action, message string, isFailed bool) { m.lock.Lock() defer m.lock.Unlock() m.Logs = append(m.Logs, NewSchedLog(candidate, action, message, isFailed)) -} +}*/ func (m *SchedLogManager) Appends(logs []SchedLog) { m.lock.Lock() @@ -336,7 +355,7 @@ func (m *SchedLogManager) Read() []string { } } - newLog := NewSchedLog(log.Candidate, strings.Join(actions, ","), log.Message, isFailed) + newLog := NewSchedLog(log.Candidate, strings.Join(actions, ","), log.Messages, isFailed) return newLog.String() } @@ -346,7 +365,7 @@ func (m *SchedLogManager) Read() []string { startIndex = index } else { log0, log := m.Logs[startIndex], m.Logs[index] - if log0.Candidate != log.Candidate || log0.Message != log.Message { + if log0.Candidate != log.Candidate || log0.Messages.String() != log.Messages.String() { rets = append(rets, joinLogs(startIndex, index)) startIndex = index } diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 2731bc472e..d7c0c9e7a9 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -507,31 +507,27 @@ func unitFitsOnCandidate( toLog := func(fit bool, reasons []PredicateFailureReason, err error, stage string) SchedLog { var ( - sFit string - message string + //sFit string + messages = make([]*LogMessage, 0) ) - if fit { + /*if fit { sFit = "Success." } else { sFit = "Failed:" - } + }*/ if err != nil { - message = fmt.Sprintf("%v", err) + messages = append(messages, &LogMessage{Type: "error", Info: fmt.Sprintf("%v", err)}) } else { - if len(reasons) == 0 { - message = "" - } else { - ss := make([]string, 0, len(reasons)) + if len(reasons) != 0 { for _, reason := range reasons { - ss = append(ss, reason.GetReason()) + messages = append(messages, &LogMessage{Type: reason.GetType(), Info: reason.GetReason()}) } - message = strings.Join(ss, ", ") } } candidateLogIndex := fmt.Sprintf("%v:%s", candidate.Getter().Name(), candidate.IndexKey()) - return NewSchedLog(candidateLogIndex, stage, fmt.Sprintf("%v %v", sFit, message), !fit) + return NewSchedLog(candidateLogIndex, stage, messages, !fit) } for _, predicate := range predicates { diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 5aa8f93df1..adda5d0caa 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -148,10 +148,15 @@ type FitPredicate interface { Execute(*Unit, Candidater) (bool, []PredicateFailureReason, error) } -type PredicateFailureReason interface { +type PredicateFailureError interface { GetReason() string } +type PredicateFailureReason interface { + PredicateFailureError + GetType() string +} + type PriorityPreFunction func(*Unit, []Candidater) (bool, []PredicateFailureReason, error) // PriorityMapFunction is a function that computes per-resource results for a given resource. diff --git a/pkg/scheduler/handler/forecast_helper.go b/pkg/scheduler/handler/forecast_helper.go index 78ec024417..13fc60ee83 100644 --- a/pkg/scheduler/handler/forecast_helper.go +++ b/pkg/scheduler/handler/forecast_helper.go @@ -16,6 +16,7 @@ package handler import ( "fmt" + "yunion.io/x/log" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/scheduler/api" @@ -51,14 +52,18 @@ func transToSchedForecastResult(result *core.SchedResultItemList) interface{} { } addInfos := func(logs core.SchedLogList, item *core.SchedResultItem) { for preName, cnt := range item.CapacityDetails { - if cnt <= 0 { - info, exist := getOrNewFilter(preName) + if cnt > 0 { + continue + } + failedLog := logs.Get(logIndex(item)) + if failedLog == nil { + log.Errorf("predicate %q count is 0, but not found failed log", preName) + continue + } + for _, msg := range failedLog.Messages { + info, exist := getOrNewFilter(msg.Type) info.Count++ - var msg string - if failedLog := logs.Get(logIndex(item)); failedLog != nil { - msg = failedLog.String() - } - info.Messages = append(info.Messages, msg) + info.Messages = append(info.Messages, msg.Info) if !exist { filters = append(filters, info) }