mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-06-20 10:22:22 +08:00
438 lines
12 KiB
Go
438 lines
12 KiB
Go
// Copyright 2019 Yunion
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package misc
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"yunion.io/x/log"
|
|
|
|
"yunion.io/x/onecloud/pkg/cloudmon/options"
|
|
"yunion.io/x/onecloud/pkg/util/influxdb"
|
|
)
|
|
|
|
type SHttpStats struct {
|
|
HttpCode2xx float64 `json:"duration.2XX"`
|
|
HttpCode4xx float64 `json:"duration.4XX"`
|
|
HttpCode5xx float64 `json:"duration.5XX"`
|
|
HitHttpCode2xx int64 `json:"hit.2XX"`
|
|
HitHttpCode4xx int64 `json:"hit.4XX"`
|
|
HitHttpCode5xx int64 `json:"hit.5XX"`
|
|
|
|
Method string `json:"method"`
|
|
Name string `json:"name"`
|
|
Path string `json:"path"`
|
|
}
|
|
|
|
type sApiHttpStats struct {
|
|
SHttpStats
|
|
|
|
Paths []SHttpStats `json:"paths"`
|
|
}
|
|
|
|
func (apiStats *sApiHttpStats) convertSnapshot(now time.Time) *sHttpStatsSnapshot {
|
|
snapshot := &sHttpStatsSnapshot{
|
|
snapshotAt: now,
|
|
stats: map[string]*SHttpStats{},
|
|
}
|
|
apiStats.SHttpStats.Method = "any"
|
|
apiStats.SHttpStats.Path = "any"
|
|
apiStats.SHttpStats.Name = "any"
|
|
snapshot.stats[getStatsKey(apiStats.SHttpStats.Method, apiStats.SHttpStats.Path, apiStats.SHttpStats.Name)] = &apiStats.SHttpStats
|
|
for i := range apiStats.Paths {
|
|
pathStats := apiStats.Paths[i]
|
|
snapshot.stats[getStatsKey(pathStats.Method, pathStats.Path, pathStats.Name)] = &pathStats
|
|
}
|
|
return snapshot
|
|
}
|
|
|
|
type sHttpStatsExt struct {
|
|
Duration2xx float64 `json:"duration_2xx"`
|
|
Duration4xx float64 `json:"duration_4xx"`
|
|
Duration5xx float64 `json:"duration_5xx"`
|
|
Hit2xx int64 `json:"hit_2xx"`
|
|
Hit4xx int64 `json:"hit_4xx"`
|
|
Hit5xx int64 `json:"hit_5xx"`
|
|
|
|
Duration2xxDiff float64 `json:"duration_2xx_diff"`
|
|
Duration4xxDiff float64 `json:"duration_4xx_diff"`
|
|
Duration5xxDiff float64 `json:"duration_5xx_diff"`
|
|
Hit2xxDiff int64 `json:"hit_2xx_diff"`
|
|
Hit4xxDiff int64 `json:"hit_4xx_diff"`
|
|
Hit5xxDiff int64 `json:"hit_5xx_diff"`
|
|
|
|
Method string `json:"method"`
|
|
Path string `json:"path"`
|
|
Name string `json:"name"`
|
|
|
|
HasDiff bool `json:"has_diff"`
|
|
}
|
|
|
|
func (v sHttpStatsExt) DelayMs2xx() float64 {
|
|
if v.Hit2xxDiff > 0 {
|
|
return v.Duration2xxDiff / float64(v.Hit2xxDiff)
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Qps2xx(interval time.Duration) float64 {
|
|
if interval > 0 {
|
|
return float64(v.Hit2xxDiff) / interval.Seconds()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) DelayMs4xx() float64 {
|
|
if v.Hit4xxDiff > 0 {
|
|
return v.Duration4xxDiff / float64(v.Hit4xxDiff)
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Qps4xx(interval time.Duration) float64 {
|
|
if interval > 0 {
|
|
return float64(v.Hit4xxDiff) / interval.Seconds()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) DelayMs5xx() float64 {
|
|
if v.Hit5xxDiff > 0 {
|
|
return v.Duration5xxDiff / float64(v.Hit5xxDiff)
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Qps5xx(interval time.Duration) float64 {
|
|
if interval > 0 {
|
|
return float64(v.Hit5xxDiff) / interval.Seconds()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Qps(interval time.Duration) float64 {
|
|
if interval > 0 {
|
|
return float64(v.Hit2xxDiff+v.Hit4xxDiff+v.Hit5xxDiff) / interval.Seconds()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) DelayMs() float64 {
|
|
if v.HitDiff() > 0 {
|
|
return v.DurationMsDiff() / float64(v.HitDiff())
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) DurationMs() float64 {
|
|
return v.Duration2xx + v.Duration4xx + v.Duration5xx
|
|
}
|
|
|
|
func (v sHttpStatsExt) Hit() int64 {
|
|
return v.Hit2xx + v.Hit4xx + v.Hit5xx
|
|
}
|
|
|
|
func (v sHttpStatsExt) DurationMsDiff() float64 {
|
|
return v.Duration2xxDiff + v.Duration4xxDiff + v.Duration5xxDiff
|
|
}
|
|
|
|
func (v sHttpStatsExt) HitDiff() int64 {
|
|
return v.Hit2xxDiff + v.Hit4xxDiff + v.Hit5xxDiff
|
|
}
|
|
|
|
func (v sHttpStatsExt) Percent2xx() float64 {
|
|
if v.DurationMsDiff() > 0 {
|
|
return v.Duration2xxDiff * 100 / v.DurationMsDiff()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) PercentHit2xx() float64 {
|
|
if v.HitDiff() > 0 {
|
|
return float64(v.Hit2xxDiff) * 100 / float64(v.HitDiff())
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Percent4xx() float64 {
|
|
if v.DurationMsDiff() > 0 {
|
|
return v.Duration4xxDiff * 100 / v.DurationMsDiff()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) PercentHit4xx() float64 {
|
|
if v.HitDiff() > 0 {
|
|
return float64(v.Hit4xxDiff) * 100 / float64(v.HitDiff())
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) Percent5xx() float64 {
|
|
if v.DurationMsDiff() > 0 {
|
|
return v.Duration5xxDiff * 100 / v.DurationMsDiff()
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func (v sHttpStatsExt) PercentHit5xx() float64 {
|
|
if v.HitDiff() > 0 {
|
|
return float64(v.Hit5xxDiff) * 100 / float64(v.HitDiff())
|
|
}
|
|
return -1
|
|
}
|
|
|
|
type sHttpStatsSnapshot struct {
|
|
snapshotAt time.Time
|
|
stats map[string]*SHttpStats
|
|
}
|
|
|
|
type sHttpStatsDiff struct {
|
|
snapshotAt time.Time
|
|
interval time.Duration
|
|
stats map[string]*sHttpStatsExt
|
|
}
|
|
|
|
func calculateHttpStatsDiff(prevSnap *sHttpStatsSnapshot, nowSnap *sHttpStatsSnapshot) *sHttpStatsDiff {
|
|
diff := sHttpStatsDiff{
|
|
snapshotAt: nowSnap.snapshotAt,
|
|
stats: map[string]*sHttpStatsExt{},
|
|
}
|
|
if prevSnap != nil {
|
|
rootKey := getStatsKey("any", "any", "any")
|
|
prevRoot := prevSnap.stats[rootKey]
|
|
nowRoot := nowSnap.stats[rootKey]
|
|
|
|
if prevRoot.HttpCode2xx > nowRoot.HttpCode2xx || prevRoot.HttpCode4xx > nowRoot.HttpCode4xx || prevRoot.HttpCode5xx > nowRoot.HttpCode5xx ||
|
|
prevRoot.HitHttpCode2xx > nowRoot.HitHttpCode2xx || prevRoot.HitHttpCode4xx > nowRoot.HitHttpCode4xx || prevRoot.HitHttpCode5xx > nowRoot.HitHttpCode5xx {
|
|
// detect a reset, skip this round
|
|
prevSnap = nil
|
|
}
|
|
}
|
|
if prevSnap != nil {
|
|
diff.interval = nowSnap.snapshotAt.Sub(prevSnap.snapshotAt)
|
|
} else {
|
|
intvMin := options.Options.CollectServiceMetricIntervalMinute
|
|
if intvMin <= 0 {
|
|
intvMin = 1
|
|
}
|
|
diff.interval = time.Duration(intvMin) * time.Minute
|
|
}
|
|
|
|
for k := range nowSnap.stats {
|
|
v := nowSnap.stats[k]
|
|
diffStats := sHttpStatsExt{
|
|
Duration2xx: v.HttpCode2xx,
|
|
Duration4xx: v.HttpCode4xx,
|
|
Duration5xx: v.HttpCode5xx,
|
|
Hit2xx: v.HitHttpCode2xx,
|
|
Hit4xx: v.HitHttpCode4xx,
|
|
Hit5xx: v.HitHttpCode5xx,
|
|
|
|
Method: v.Method,
|
|
Path: v.Path,
|
|
Name: v.Name,
|
|
}
|
|
if prevSnap != nil {
|
|
if prevStats, ok := prevSnap.stats[k]; ok {
|
|
diffStats.HasDiff = true
|
|
diffStats.Duration2xxDiff = v.HttpCode2xx - prevStats.HttpCode2xx
|
|
diffStats.Duration4xxDiff = v.HttpCode4xx - prevStats.HttpCode4xx
|
|
diffStats.Duration5xxDiff = v.HttpCode5xx - prevStats.HttpCode5xx
|
|
diffStats.Hit2xxDiff = v.HitHttpCode2xx - prevStats.HitHttpCode2xx
|
|
diffStats.Hit4xxDiff = v.HitHttpCode4xx - prevStats.HitHttpCode4xx
|
|
diffStats.Hit5xxDiff = v.HitHttpCode5xx - prevStats.HitHttpCode5xx
|
|
}
|
|
}
|
|
|
|
diff.stats[k] = &diffStats
|
|
}
|
|
return &diff
|
|
}
|
|
|
|
var (
|
|
httpStatsSnapshot = map[string]*sHttpStatsSnapshot{}
|
|
)
|
|
|
|
func getStatsKey(method, path, name string) string {
|
|
return fmt.Sprintf("%s.%s.%s", method, path, name)
|
|
}
|
|
|
|
func getSnapshotKey(serviceName, url string) string {
|
|
return fmt.Sprintf("%s.%s", serviceName, url)
|
|
}
|
|
|
|
func updateHttpStatsSnapshot(serviceName string, url string, now time.Time, apiStats sApiHttpStats, serviceType, regionId, version string) []influxdb.SMetricData {
|
|
snapshot := apiStats.convertSnapshot(now)
|
|
|
|
snapshotKey := getSnapshotKey(serviceName, url)
|
|
var vdiffStats *sHttpStatsDiff
|
|
if prevSnap, ok := httpStatsSnapshot[snapshotKey]; ok {
|
|
vdiffStats = calculateHttpStatsDiff(prevSnap, snapshot)
|
|
} else {
|
|
// no prev records, just add
|
|
vdiffStats = calculateHttpStatsDiff(nil, snapshot)
|
|
}
|
|
httpStatsSnapshot[snapshotKey] = snapshot
|
|
|
|
metrics := vdiffStats.metrics(serviceName, serviceType, regionId, version)
|
|
|
|
log.Debugf("updateHttpStatsSnapshot %s %s snapshotAt: %s diffAt: %s intval: %f metrics: %d", serviceName, url, snapshot.snapshotAt, vdiffStats.snapshotAt, vdiffStats.interval.Seconds(), len(metrics))
|
|
|
|
return metrics
|
|
}
|
|
|
|
func appendMetric(metrics []influxdb.SKeyValue, key string, v float64) []influxdb.SKeyValue {
|
|
if v >= 0 {
|
|
metrics = append(metrics, influxdb.SKeyValue{
|
|
Key: key,
|
|
Value: fmt.Sprintf("%f", v),
|
|
})
|
|
}
|
|
return metrics
|
|
}
|
|
|
|
func (diff *sHttpStatsDiff) metrics(service, serviceType, regionId, version string) []influxdb.SMetricData {
|
|
metrics := make([]influxdb.SMetricData, 0)
|
|
genTags := func(v *sHttpStatsExt) []influxdb.SKeyValue {
|
|
return []influxdb.SKeyValue{
|
|
{
|
|
Key: "service",
|
|
Value: service,
|
|
},
|
|
{
|
|
Key: "service_type",
|
|
Value: serviceType,
|
|
},
|
|
{
|
|
Key: "region",
|
|
Value: regionId,
|
|
},
|
|
{
|
|
Key: "version",
|
|
Value: version,
|
|
},
|
|
{
|
|
Key: "method",
|
|
Value: v.Method,
|
|
},
|
|
{
|
|
Key: "path",
|
|
Value: v.Path,
|
|
},
|
|
{
|
|
Key: "name",
|
|
Value: v.Name,
|
|
},
|
|
{
|
|
Key: "interval_secs",
|
|
Value: fmt.Sprintf("%f", diff.interval.Seconds()),
|
|
},
|
|
}
|
|
}
|
|
for k := range diff.stats {
|
|
v := diff.stats[k]
|
|
|
|
// ignore stats with no hit
|
|
if v.Hit() <= 0 {
|
|
continue
|
|
}
|
|
// ignore stats with no diff
|
|
if v.HasDiff && v.HitDiff() <= 0 {
|
|
continue
|
|
}
|
|
|
|
metric := influxdb.SMetricData{
|
|
Name: METIRCY_TYPE_HTTP_REQUST,
|
|
Timestamp: diff.snapshotAt,
|
|
Tags: genTags(v),
|
|
Metrics: []influxdb.SKeyValue{
|
|
{
|
|
Key: "duration_ms_any",
|
|
Value: fmt.Sprintf("%f", v.DurationMs()),
|
|
},
|
|
{
|
|
Key: "hit_any",
|
|
Value: fmt.Sprintf("%d", v.Hit()),
|
|
},
|
|
},
|
|
}
|
|
if v.HasDiff {
|
|
metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_any", v.DurationMsDiff())
|
|
metric.Metrics = appendMetric(metric.Metrics, "hit_delta_any", float64(v.HitDiff()))
|
|
metric.Metrics = appendMetric(metric.Metrics, "delay_ms_any", v.DelayMs())
|
|
metric.Metrics = appendMetric(metric.Metrics, "qps_any", v.Qps(diff.interval))
|
|
}
|
|
metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{
|
|
{
|
|
Key: "duration_ms_2xx",
|
|
Value: fmt.Sprintf("%f", v.Duration2xx),
|
|
},
|
|
{
|
|
Key: "hit_2xx",
|
|
Value: fmt.Sprintf("%d", v.Hit2xx),
|
|
},
|
|
}...)
|
|
if v.HasDiff {
|
|
metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_2xx", v.Duration2xxDiff)
|
|
metric.Metrics = appendMetric(metric.Metrics, "hit_delta_2xx", float64(v.Hit2xxDiff))
|
|
metric.Metrics = appendMetric(metric.Metrics, "delay_ms_2xx", v.DelayMs2xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_hit_2xx", v.PercentHit2xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_duration_2xx", v.Percent2xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "qps_2xx", v.Qps2xx(diff.interval))
|
|
}
|
|
metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{
|
|
{
|
|
Key: "duration_ms_4xx",
|
|
Value: fmt.Sprintf("%f", v.Duration4xx),
|
|
},
|
|
{
|
|
Key: "hit_4xx",
|
|
Value: fmt.Sprintf("%d", v.Hit4xx),
|
|
},
|
|
}...)
|
|
if v.HasDiff {
|
|
metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_4xx", v.Duration4xxDiff)
|
|
metric.Metrics = appendMetric(metric.Metrics, "hit_delta_4xx", float64(v.Hit4xxDiff))
|
|
metric.Metrics = appendMetric(metric.Metrics, "delay_ms_4xx", v.DelayMs4xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_hit_4xx", v.PercentHit4xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_duration_4xx", v.Percent4xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "qps_4xx", v.Qps4xx(diff.interval))
|
|
}
|
|
metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{
|
|
{
|
|
Key: "duration_ms_5xx",
|
|
Value: fmt.Sprintf("%f", v.Duration5xx),
|
|
},
|
|
{
|
|
Key: "hit_5xx",
|
|
Value: fmt.Sprintf("%d", v.Hit5xx),
|
|
},
|
|
}...)
|
|
if v.HasDiff {
|
|
metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_5xx", v.Duration5xxDiff)
|
|
metric.Metrics = appendMetric(metric.Metrics, "hit_delta_5xx", float64(v.Hit5xxDiff))
|
|
metric.Metrics = appendMetric(metric.Metrics, "delay_ms_5xx", v.DelayMs5xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_hit_5xx", v.PercentHit5xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "percent_duration_5xx", v.Percent5xx())
|
|
metric.Metrics = appendMetric(metric.Metrics, "qps_5xx", v.Qps5xx(diff.interval))
|
|
}
|
|
|
|
metrics = append(metrics, metric)
|
|
}
|
|
return metrics
|
|
}
|