From d17fbaf02c8a7d08fca891708a46d32f2946860d Mon Sep 17 00:00:00 2001 From: Zexi Date: Fri, 21 Jun 2019 13:37:26 +0800 Subject: [PATCH] scheduler: fix prefer avoid score --- pkg/apis/compute/api.go | 1 + .../predicates/aggregate_predicate.go | 12 +- .../algorithm/predicates/schedtag_helper.go | 26 ++- .../priorities/guest/avoid_same_host.go | 2 +- .../algorithm/priorities/guest/capacity.go | 2 +- .../algorithm/priorities/guest/creating.go | 2 +- .../algorithm/priorities/priorities.go | 26 +-- pkg/scheduler/core/context.go | 21 +- pkg/scheduler/core/generic_scheduler.go | 4 +- pkg/scheduler/core/score/score.go | 210 ++++-------------- pkg/scheduler/core/score/score_test.go | 136 ++---------- pkg/scheduler/core/types.go | 17 +- pkg/scheduler/manager/task_queue.go | 4 +- 13 files changed, 116 insertions(+), 347 deletions(-) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index c6c559e5d9..4b0dd4e8c8 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -25,6 +25,7 @@ type SchedtagConfig struct { Id string `json:"id"` Strategy string `json:"strategy"` + Weight int `json:"weight"` } type NetworkConfig struct { diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go index d403f24045..b12a37353a 100644 --- a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -110,16 +110,14 @@ func (p *AggregatePredicate) exec(h *PredicateHelper) string { return "" } -func SetCandidateScoreBySchedtag(u *core.Unit, c core.Candidater, aggCountMap map[string]int, postiveScore bool) { +func SetCandidateScoreBySchedtag(u *core.Unit, c core.Candidater, aggCountMap map[string]int, prefer bool) { stepScore := core.PriorityStep - if !postiveScore { - stepScore = -stepScore + doSet := u.SetPreferScore + if !prefer { + doSet = u.SetAvoidScore } for n, count := range aggCountMap { - u.SetFrontScore( - c.IndexKey(), - score.NewScore(score.TScore(count*stepScore), n), - ) + doSet(c.IndexKey(), score.NewScore(score.TScore(count*stepScore), n)) } } diff --git a/pkg/scheduler/algorithm/predicates/schedtag_helper.go b/pkg/scheduler/algorithm/predicates/schedtag_helper.go index 0f7a624003..42d188122d 100644 --- a/pkg/scheduler/algorithm/predicates/schedtag_helper.go +++ b/pkg/scheduler/algorithm/predicates/schedtag_helper.go @@ -83,18 +83,25 @@ func (p *SchedtagPredicate) Check(candidate ISchedtagCandidate) error { func GetSchedtagCount(inTags []computeapi.SchedtagConfig, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { countMap = make(map[string]int) - in := func(objTag models.SSchedtag, inTags []computeapi.SchedtagConfig) bool { + in := func(objTag models.SSchedtag, inTags []computeapi.SchedtagConfig) (bool, int) { for _, tag := range inTags { if tag.Id == objTag.Id || tag.Id == objTag.Name { - return true + return true, tag.Weight } } - return false + return false, 0 } for _, objTag := range objTags { - if in(objTag, inTags) { - countMap[fmt.Sprintf("%s:%s:%s", objTag.Id, objTag.Name, strategy)]++ + if ok, weight := in(objTag, inTags); ok { + key := fmt.Sprintf("%s:%s:%s", objTag.Id, objTag.Name, strategy) + score, ok := countMap[key] + if ok { + score += weight + } else { + score = weight + } + countMap[key] = score } } return @@ -116,7 +123,10 @@ func GetRequestSchedtags(reqTags []*computeapi.SchedtagConfig, allTags []models. appendedTagIds := make(map[string]int) - appendTagByStrategy := func(tag *computeapi.SchedtagConfig) { + appendTagByStrategy := func(tag *computeapi.SchedtagConfig, defaultWeight int) { + if tag.Weight <= 0 { + tag.Weight = defaultWeight + } switch tag.Strategy { case models.STRATEGY_REQUIRE: requireTags = append(requireTags, *tag) @@ -130,7 +140,7 @@ func GetRequestSchedtags(reqTags []*computeapi.SchedtagConfig, allTags []models. } for _, tag := range reqTags { - appendTagByStrategy(tag) + appendTagByStrategy(tag, 10) appendedTagIds[tag.Id] = 1 } @@ -141,7 +151,7 @@ func GetRequestSchedtags(reqTags []*computeapi.SchedtagConfig, allTags []models. if !(nameOk || idOk) { apiTag := &computeapi.SchedtagConfig{Id: tag.Id, Strategy: tag.DefaultStrategy} - appendTagByStrategy(apiTag) + appendTagByStrategy(apiTag, 1) } } diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go index ab528565e5..2b25f44cea 100644 --- a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go @@ -36,7 +36,7 @@ func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostP ownerTenantID := u.SchedData().Project if count, ok := c.Getter().ProjectGuests()[ownerTenantID]; ok && count > 0 { - h.SetFrontRawScore(-1 * int(count)) + h.SetScore(-1 * int(count)) } return h.GetResult() diff --git a/pkg/scheduler/algorithm/priorities/guest/capacity.go b/pkg/scheduler/algorithm/priorities/guest/capacity.go index 8c99e9cd58..a614937c2a 100644 --- a/pkg/scheduler/algorithm/priorities/guest/capacity.go +++ b/pkg/scheduler/algorithm/priorities/guest/capacity.go @@ -35,7 +35,7 @@ func (p *CapacityPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori h := priorities.NewPriorityHelper(p, u, c) capacity := u.GetCapacity(c.IndexKey()) - h.SetRawScore(int(capacity)) + h.SetScore(int(capacity)) return h.GetResult() } diff --git a/pkg/scheduler/algorithm/priorities/guest/creating.go b/pkg/scheduler/algorithm/priorities/guest/creating.go index 78b47334a8..da14e10187 100644 --- a/pkg/scheduler/algorithm/priorities/guest/creating.go +++ b/pkg/scheduler/algorithm/priorities/guest/creating.go @@ -37,7 +37,7 @@ func (p *CreatingPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori creatingGuestCount := c.Getter().CreatingGuestCount() if creatingGuestCount > 0 { score := -int(creatingGuestCount) - h.SetFrontScore(score) + h.SetScore(score) } return h.GetResult() diff --git a/pkg/scheduler/algorithm/priorities/priorities.go b/pkg/scheduler/algorithm/priorities/priorities.go index b55958f0cf..1945256cb3 100644 --- a/pkg/scheduler/algorithm/priorities/priorities.go +++ b/pkg/scheduler/algorithm/priorities/priorities.go @@ -36,13 +36,6 @@ func NewPriorityHelper(p core.Priority, u *core.Unit, c core.Candidater) *Priori } } -func (h *PriorityHelper) setIntervalScore(val int) score.SScore { - h.score = score.NewScore( - h.priority.ScoreIntervals().ToScore(int64(val)), - h.priority.Name()) - return h.score -} - func (h *PriorityHelper) setRawScore(val int) score.SScore { h.score = score.NewScore( score.TScore(val), @@ -51,23 +44,18 @@ func (h *PriorityHelper) setRawScore(val int) score.SScore { } func (h *PriorityHelper) SetScore(val int) { - h.setIntervalScore(val) - h.unit.SetScore(h.Candidate.IndexKey(), h.score) -} - -func (h *PriorityHelper) SetFrontScore(val int) { - h.setIntervalScore(val) - h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score) -} - -func (h *PriorityHelper) SetRawScore(val int) { h.setRawScore(val) h.unit.SetScore(h.Candidate.IndexKey(), h.score) } -func (h *PriorityHelper) SetFrontRawScore(val int) { +func (h *PriorityHelper) SetPreferScore(val int) { h.setRawScore(val) - h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score) + h.unit.SetPreferScore(h.Candidate.IndexKey(), h.score) +} + +func (h *PriorityHelper) SetAvoidScore(val int) { + h.setRawScore(val) + h.unit.SetAvoidScore(h.Candidate.IndexKey(), h.score) } func (h *PriorityHelper) SetError(err error) { diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index 6c1bd013c1..a128bf6d87 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -19,6 +19,7 @@ import ( "sort" "strings" "sync" + "yunion.io/x/pkg/tristate" "yunion.io/x/log" @@ -209,7 +210,7 @@ func newScore() *Score { func newZeroScore() Score { s := newScore() - s.Append(score.NewZeroScore()) + s.SetScore(score.NewZeroScore(), tristate.None) return *s } @@ -554,7 +555,7 @@ type ScoreValue struct { value score.TScore } -func (u *Unit) setScore(id string, val score.SScore, tofront bool) { +func (u *Unit) setScore(id string, val score.SScore, prefer tristate.TriState) { u.scoreLock.Lock() defer u.scoreLock.Unlock() @@ -568,21 +569,21 @@ func (u *Unit) setScore(id string, val score.SScore, tofront bool) { u.ScoreMap[id] = scoreObj } - if tofront { - scoreObj.AddToFirst(val) - } else { - scoreObj.SetScore(val) - } + scoreObj.ScoreBucket.SetScore(val, prefer) log.V(10).Infof("SetScore: %q -> %s", id, val.String()) } func (u *Unit) SetScore(id string, val score.SScore) { - u.setScore(id, val, false) + u.setScore(id, val, tristate.None) } -func (u *Unit) SetFrontScore(id string, val score.SScore) { - u.setScore(id, val, true) +func (u *Unit) SetPreferScore(id string, val score.SScore) { + u.setScore(id, val, tristate.True) +} + +func (u *Unit) SetAvoidScore(id string, val score.SScore) { + u.setScore(id, val, tristate.False) } func (u *Unit) GetScore(id string) Score { diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index df965de09f..2731bc472e 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -180,7 +180,7 @@ func newSchedResultByCtx(u *Unit, count int64, c Candidater) *SchedResultItem { Count: count, Capacity: u.GetCapacity(id), Name: c.Getter().Name(), - Score: u.GetScore(id).DigitString(), + Score: u.GetScore(id).String(), Data: u.GetFiltedData(id, count), Candidater: c, AllocatedResource: u.GetAllocatedResource(id), @@ -652,7 +652,7 @@ func PrioritizeCandidates( } if log.V(10) { for i := range result { - log.Infof("Host %s => Score %s", result[i].Host, result[i].Score.DigitString()) + log.Infof("Host %s => Score %s", result[i].Host, result[i].Score.String()) } } return result, nil diff --git a/pkg/scheduler/core/score/score.go b/pkg/scheduler/core/score/score.go index f591a9a772..4f675712f1 100644 --- a/pkg/scheduler/core/score/score.go +++ b/pkg/scheduler/core/score/score.go @@ -15,10 +15,11 @@ package score import ( - "container/list" "fmt" "math" - //"yunion.io/x/log" + "strings" + + "yunion.io/x/pkg/tristate" ) type TScore int @@ -68,200 +69,67 @@ func (v SScore) String() string { return fmt.Sprintf("%s: %d", v.Name, v.Score) } -type Scores struct { - scores *list.List -} +type scores map[string]int -func newScores() *Scores { - return &Scores{ - scores: list.New(), +func (ss scores) Total() int { + ret := 0 + for _, s := range ss { + ret += s } -} - -func (s *Scores) Append(scores ...SScore) *Scores { - for _, score := range scores { - s.scores.PushBack(score) - } - return s -} - -func (s *Scores) AddToFirst(score SScore) *Scores { - s.scores.PushFront(score) - return s -} - -func (s *Scores) Range(iterFunc func(ele *list.Element, score SScore) bool) { - for ele := s.scores.Front(); ele != nil; ele = ele.Next() { - cont := iterFunc(ele, ele.Value.(SScore)) - if !cont { - break - } - } -} - -func (s *Scores) SetScore(score SScore) *Scores { - exists := false - rf := func(ele *list.Element, oscore SScore) bool { - if oscore.Name == score.Name { - exists = true - oscore.Score = score.Score - ele.Value = oscore - return false - } - return true - } - s.Range(rf) - if !exists { - s.Append(score) - } - return s -} - -func (s *Scores) AddScore(score SScore) *Scores { - exists := false - rf := func(ele *list.Element, oscore SScore) bool { - if oscore.Name == score.Name { - exists = true - oscore.Score += score.Score - ele.Value = oscore - return false - } - return true - } - s.Range(rf) - if !exists { - s.Append(score) - } - return s -} - -func (s *Scores) Len() int { - return s.scores.Len() -} - -func (s *Scores) GetScores() []SScore { - ret := make([]SScore, 0) - rf := func(_ *list.Element, score SScore) bool { - ret = append(ret, score) - return true - } - s.Range(rf) return ret } type ScoreBucket struct { - scores *Scores + preferScore scores + avoidScore scores + normalScore scores } func NewScoreBuckets() *ScoreBucket { return &ScoreBucket{ - scores: newScores(), + preferScore: make(map[string]int), + avoidScore: make(map[string]int), + normalScore: make(map[string]int), } } -func (b *ScoreBucket) AddToFirst(score SScore) *ScoreBucket { - b.scores.AddToFirst(score) +func (b *ScoreBucket) SetScore(score SScore, prefer tristate.TriState) *ScoreBucket { + scoreToSet := b.normalScore + if prefer.IsTrue() { + scoreToSet = b.preferScore + } else if prefer.IsFalse() { + scoreToSet = b.avoidScore + } + scoreToSet[score.Name] = int(score.Score) return b } -func (b *ScoreBucket) Append(scores ...SScore) *ScoreBucket { - b.scores.Append(scores...) - return b +func PreferLess(b1, b2 *ScoreBucket) bool { + return b1.preferScore.Total() < b2.preferScore.Total() } -func (b *ScoreBucket) GetScores() []SScore { - return b.scores.GetScores() +func AvoidLess(b1, b2 *ScoreBucket) bool { + return b1.avoidScore.Total() < b2.avoidScore.Total() } -func (b *ScoreBucket) SetScore(score SScore) *ScoreBucket { - b.scores.SetScore(score) - return b +func NormalLess(b1, b2 *ScoreBucket) bool { + return b1.normalScore.Total() < b2.normalScore.Total() } -func (b *ScoreBucket) AddScore(score SScore) *ScoreBucket { - b.scores.AddScore(score) - return b -} - -func (b *ScoreBucket) GetScore(scoreName string) (int, SScore) { - for i, oscore := range b.scores.GetScores() { - if oscore.Name == scoreName { - return i, oscore - } - } - return -1, SScore{} -} - -func (b *ScoreBucket) Len() int { - return b.scores.Len() -} - -func (b *ScoreBucket) DigitString() string { - s := "" - rf := func(_ *list.Element, score SScore) bool { - s = fmt.Sprintf("%s%d", s, score.Score) - return true - } - b.scores.Range(rf) - return s -} - -func extend(scores []SScore, length int) []SScore { - olen := len(scores) - if olen >= length { - return scores - } - ret := make([]SScore, 0) - zeroDigits := length - olen - for i := 0; i < zeroDigits; i++ { - ret = append(ret, NewZeroScore()) - } - ret = append(ret, scores...) - return ret -} - -func Equal(b1, b2 *ScoreBucket) bool { - return compare(b1, b2, func(s1, s2 TScore) bool { return s1 == s2 }) -} - -func Less(b1, b2 *ScoreBucket) bool { - return compare(b1, b2, func(s1, s2 TScore) bool { return s1 < s2 }) -} - -func compare(b1, b2 *ScoreBucket, cf func(s1, s2 TScore) bool) bool { - maxLen := int(math.Max(float64(b1.Len()), float64(b2.Len()))) - s1 := b1.GetScores() - s2 := b2.GetScores() - s1 = extend(s1, maxLen) - s2 = extend(s2, maxLen) - for i := range s1 { - v1 := s1[i].GetScore() - v2 := s2[i].GetScore() - ok := cf(v1, v2) - if ok { - return true - } else if !ok { - return false - } - } - return false -} - -func (b *ScoreBucket) debugString(vals []SScore, ret string) string { - if len(vals) == 0 { - return ret - } - restVal := vals[1:] - if len(restVal) == 0 { - return vals[0].String() - } - str := b.debugString(restVal, ret) - str = fmt.Sprintf("%s, %s", vals[0].String(), str) - return str +func (b *ScoreBucket) debugString(kind string, vals map[string]int) string { + return fmt.Sprintf("%s: %v", kind, vals) } func (b *ScoreBucket) String() string { - return b.debugString(b.GetScores(), "") + kinds := make([]string, 0) + for kind, ss := range map[string]map[string]int{ + "prefer": b.preferScore, + "avoid": b.avoidScore, + "normal": b.normalScore, + } { + kinds = append(kinds, b.debugString(kind, ss)) + } + return strings.Join(kinds, "\n") } type Interval struct { diff --git a/pkg/scheduler/core/score/score_test.go b/pkg/scheduler/core/score/score_test.go index 3a84c28122..522a4af35b 100644 --- a/pkg/scheduler/core/score/score_test.go +++ b/pkg/scheduler/core/score/score_test.go @@ -16,58 +16,15 @@ package score import ( "testing" -) -func TestScoreBucket_String(t *testing.T) { - type fields struct { - scores *Scores - } - tests := []struct { - name string - fields fields - want string - }{ - { - name: "EmptyScores", - fields: fields{newScores()}, - want: "", - }, - { - name: "Scores100", - fields: fields{newScores().Append( - NewMidScore("mid"), - NewZeroScore(), - NewZeroScore(), - )}, - want: "mid: 1, zero: 0, zero: 0", - }, - { - name: "Scores201-1", - fields: fields{newScores().Append( - NewMaxScore("max"), - NewZeroScore(), - NewMidScore("mid"), - NewMinScore("min"), - )}, - want: "max: 2, zero: 0, mid: 1, min: -1", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &ScoreBucket{ - scores: tt.fields.scores, - } - if got := b.String(); got != tt.want { - t.Errorf("ScoreBucket.String() = %v, want %v", got, tt.want) - } - }) - } -} + "yunion.io/x/pkg/tristate" +) func TestLess(t *testing.T) { type args struct { - b1 *ScoreBucket - b2 *ScoreBucket + b1 *ScoreBucket + b2 *ScoreBucket + lessFunc func(s1, s2 *ScoreBucket) bool } tests := []struct { name string @@ -77,94 +34,27 @@ func TestLess(t *testing.T) { { name: "equal", args: args{ - b1: NewScoreBuckets(), - b2: NewScoreBuckets(), + b1: NewScoreBuckets(), + b2: NewScoreBuckets(), + lessFunc: NormalLess, }, want: false, }, { - name: "extendEqual", + name: "less", args: args{ - b1: NewScoreBuckets().Append( - NewZeroScore(), NewMidScore("1"), - ), - b2: NewScoreBuckets().Append(NewMidScore("1")), - }, - want: false, - }, - { - name: "10<100", - args: args{ - b1: NewScoreBuckets().Append( - NewMidScore("1"), - NewZeroScore(), - ), - b2: NewScoreBuckets().Append( - NewMidScore("1"), - NewZeroScore(), - NewZeroScore(), - ), + b1: NewScoreBuckets().SetScore(SScore{1, "p1"}, tristate.True), + b2: NewScoreBuckets().SetScore(SScore{1, "p1"}, tristate.True).SetScore(SScore{2, "p2"}, tristate.True), + lessFunc: PreferLess, }, want: true, }, - { - name: "101>10", - args: args{ - b1: NewScoreBuckets().Append( - NewMidScore("1"), - NewZeroScore(), - NewMidScore("1"), - ), - b2: NewScoreBuckets().Append( - NewMidScore("1"), - NewZeroScore(), - ), - }, - want: false, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := Less(tt.args.b1, tt.args.b2); got != tt.want { + if got := tt.args.lessFunc(tt.args.b1, tt.args.b2); got != tt.want { t.Errorf("Less() = %v, want %v", got, tt.want) } }) } } - -func TestScoreBucket_DigitString(t *testing.T) { - type fields struct { - scores *Scores - } - tests := []struct { - name string - fields fields - want string - }{ - { - name: "2-101", - fields: fields{newScores().Append( - NewMaxScore(""), - NewMinScore(""), - NewZeroScore(), - NewMidScore(""), - )}, - want: "2-101", - }, - { - name: "empty", - fields: fields{newScores()}, - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &ScoreBucket{ - scores: tt.fields.scores, - } - if got := b.DigitString(); got != tt.want { - t.Errorf("ScoreBucket.DigitString() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 11a38697b0..bfcc876c54 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -116,10 +116,23 @@ func (h HostPriorityList) Len() int { } func (h HostPriorityList) Less(i, j int) bool { - if score.Equal(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket) { + s1 := h[i].Score.ScoreBucket + s2 := h[j].Score.ScoreBucket + preferLess := score.PreferLess(s1, s2) + avoidLess := score.AvoidLess(s1, s2) + normalLess := score.NormalLess(s1, s2) + + if !(preferLess || avoidLess || normalLess) { return h[i].Host < h[j].Host } - return score.Less(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket) + + if preferLess { + return true + } + if avoidLess { + return false + } + return normalLess } func (h HostPriorityList) Swap(i, j int) { diff --git a/pkg/scheduler/manager/task_queue.go b/pkg/scheduler/manager/task_queue.go index df21b6848c..2fece07c8b 100644 --- a/pkg/scheduler/manager/task_queue.go +++ b/pkg/scheduler/manager/task_queue.go @@ -263,8 +263,8 @@ type Task struct { Time time.Time SchedInfo *api.SchedInfo Consuming time.Duration - taskExecutors []*TaskExecutor - manager *SchedulerManager + taskExecutors []*TaskExecutor `json:"-"` + manager *SchedulerManager `json:"-"` lock sync.Mutex waitCh chan struct{}