Fix TM2 down caches to exclude admin_down, offline
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/commit/10129f3f Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/tree/10129f3f Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/diff/10129f3f Branch: refs/heads/master Commit: 10129f3fed1a1597ab206528bc7eb673166ffcb5 Parents: b29a5d5 Author: Robert Butts <robert.o.bu...@gmail.com> Authored: Mon Oct 31 16:08:42 2016 -0600 Committer: Dave Neuman <neu...@apache.org> Committed: Mon Nov 7 12:29:08 2016 -0700 ---------------------------------------------------------------------- .../experimental/traffic_monitor/enum/enum.go | 51 ++++++++++++++++++++ .../traffic_monitor/health/cache_health.go | 19 +++++--- .../traffic_monitor/manager/datarequest.go | 21 ++++++-- .../traffic_monitor/manager/manager.go | 1 + .../traffic_monitor/manager/opsconfig.go | 2 + 5 files changed, 84 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/10129f3f/traffic_monitor/experimental/traffic_monitor/enum/enum.go ---------------------------------------------------------------------- diff --git a/traffic_monitor/experimental/traffic_monitor/enum/enum.go b/traffic_monitor/experimental/traffic_monitor/enum/enum.go index e028a74..c8389b9 100644 --- a/traffic_monitor/experimental/traffic_monitor/enum/enum.go +++ b/traffic_monitor/experimental/traffic_monitor/enum/enum.go @@ -91,3 +91,54 @@ func DSTypeFromString(s string) DSType { return DSTypeInvalid } } + +// CacheStatus represents the Traffic Server status set in Traffic Ops (online, offline, admin_down, reported). The string values of this type should match the Traffic Ops values. +type CacheStatus string + +const ( + // CacheStatusAdminDown represents a cache which has been administratively marked as down, but which should still appear in the CDN (Traffic Server, Traffic Monitor, Traffic Router). + CacheStatusAdminDown = CacheStatus("ADMIN_DOWN") + // CacheStatusOnline represents a cache which has been marked as Online in Traffic Ops, irrespective of monitoring. Traffic Monitor will always flag these caches as available. + CacheStatusOnline = CacheStatus("ONLINE") + // CacheStatusOffline represents a cache which has been marked as Offline in Traffic Ops. These caches will not be returned in any endpoint, and Traffic Monitor acts like they don't exist. + CacheStatusOffline = CacheStatus("OFFLINE") + // CacheStatusReported represents a cache which has been marked as Reported in Traffic Ops. These caches are polled for health and returned in endpoints as available or unavailable based on bandwidth, response time, and other factors. The vast majority of caches should be Reported. + CacheStatusReported = CacheStatus("REPORTED") + // CacheStatusInvalid represents an invalid status enumeration. + CacheStatusInvalid = CacheStatus("") +) + +// String returns a string representation of this cache status +func (t CacheStatus) String() string { + switch t { + case CacheStatusAdminDown: + fallthrough + case CacheStatusOnline: + fallthrough + case CacheStatusOffline: + fallthrough + case CacheStatusReported: + return string(t) + default: + return "INVALID" + } +} + +// CacheStatusFromString returns a CacheStatus from its string representation, or CacheStatusInvalid if the string is not a valid type. +func CacheStatusFromString(s string) CacheStatus { + s = strings.ToLower(s) + switch s { + case "admin_down": + fallthrough + case "admindown": + return CacheStatusAdminDown + case "offline": + return CacheStatusOffline + case "online": + return CacheStatusOnline + case "reported": + return CacheStatusReported + default: + return CacheStatusInvalid + } +} http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/10129f3f/traffic_monitor/experimental/traffic_monitor/health/cache_health.go ---------------------------------------------------------------------- diff --git a/traffic_monitor/experimental/traffic_monitor/health/cache_health.go b/traffic_monitor/experimental/traffic_monitor/health/cache_health.go index d4c7639..63cff4f 100644 --- a/traffic_monitor/experimental/traffic_monitor/health/cache_health.go +++ b/traffic_monitor/experimental/traffic_monitor/health/cache_health.go @@ -9,6 +9,7 @@ import ( "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/common/log" "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/traffic_monitor/cache" + "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/traffic_monitor/enum" traffic_ops "github.com/apache/incubator-trafficcontrol/traffic_ops/client" ) @@ -106,15 +107,19 @@ func cacheCapacityKbps(result cache.Result) int64 { // EvalCache returns whether the given cache should be marked available, and a string describing why func EvalCache(result cache.Result, mc *traffic_ops.TrafficMonitorConfigMap) (bool, string) { toServer := mc.TrafficServer[string(result.ID)] - status := toServer.Status + status := enum.CacheStatusFromString(toServer.Status) + if status == enum.CacheStatusInvalid { + log.Errorf("Cache %v got invalid status from Traffic Ops '%v' - treating as Reported\n", result.ID, toServer.Status) + } params := mc.Profile[toServer.Profile].Parameters switch { - case status == "ADMIN_DOWN": - return false, "set to ADMIN_DOWN" - case status == "OFFLINE": - return false, "set to OFFLINE" - case status == "ONLINE": - return true, "set to ONLINE" + case status == enum.CacheStatusAdminDown: + return false, "set to " + status.String() + case status == enum.CacheStatusOffline: + log.Errorf("Cache %v set to offline, but still polled\n", result.ID) + return false, "set to " + status.String() + case status == enum.CacheStatusOnline: + return true, "set to " + status.String() case result.Error != nil: return false, fmt.Sprintf("error: %v", result.Error) case result.Vitals.LoadAvg > params.HealthThresholdLoadAvg: http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/10129f3f/traffic_monitor/experimental/traffic_monitor/manager/datarequest.go ---------------------------------------------------------------------- diff --git a/traffic_monitor/experimental/traffic_monitor/manager/datarequest.go b/traffic_monitor/experimental/traffic_monitor/manager/datarequest.go index ccb7447..735c222 100644 --- a/traffic_monitor/experimental/traffic_monitor/manager/datarequest.go +++ b/traffic_monitor/experimental/traffic_monitor/manager/datarequest.go @@ -20,6 +20,7 @@ import ( "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/traffic_monitor/srvhttp" todata "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/traffic_monitor/trafficopsdata" towrap "github.com/apache/incubator-trafficcontrol/traffic_monitor/experimental/traffic_monitor/trafficopswrapper" + to "github.com/apache/incubator-trafficcontrol/traffic_ops/client" ) // JSONEvents represents the structure we wish to serialize to JSON, for Events. @@ -432,6 +433,7 @@ func DataRequest( localCacheStatus CacheAvailableStatusThreadsafe, lastStats LastStatsThreadsafe, unpolledCaches UnpolledCachesThreadsafe, + monitorConfig TrafficMonitorConfigMapThreadsafe, ) (body []byte, responseCode int) { // handleErr takes an error, and the request type it came from, and logs. It is ok to call with a nil error, in which case this is a no-op. @@ -520,7 +522,7 @@ func DataRequest( case srvhttp.APICacheAvailableCount: return []byte(strconv.Itoa(cacheAvailableCount(localStates.Get().Caches))), http.StatusOK case srvhttp.APICacheDownCount: - return []byte(strconv.Itoa(cacheDownCount(localStates.Get().Caches))), http.StatusOK + return []byte(strconv.Itoa(cacheDownCount(localStates.Get().Caches, monitorConfig.Get().TrafficServer))), http.StatusOK case srvhttp.APIVersion: s := "traffic_monitor-" + staticAppData.Version + "." if len(staticAppData.GitRevision) > 6 { @@ -670,7 +672,8 @@ func createCacheConnections(statHistory map[enum.CacheName][]cache.Result) map[e return conns } -func cacheDownCount(caches map[enum.CacheName]peer.IsAvailable) int { +// cacheOfflineCount returns the total caches not available, including marked unavailable, status offline, and status admin_down +func cacheOfflineCount(caches map[enum.CacheName]peer.IsAvailable) int { count := 0 for _, available := range caches { if !available.IsAvailable { @@ -680,8 +683,20 @@ func cacheDownCount(caches map[enum.CacheName]peer.IsAvailable) int { return count } +// cacheAvailableCount returns the total caches available, including marked available and status online func cacheAvailableCount(caches map[enum.CacheName]peer.IsAvailable) int { - return len(caches) - cacheDownCount(caches) + return len(caches) - cacheOfflineCount(caches) +} + +// cacheOfflineCount returns the total reported caches marked down, excluding status offline and admin_down. +func cacheDownCount(caches map[enum.CacheName]peer.IsAvailable, toServers map[string]to.TrafficServer) int { + count := 0 + for cache, available := range caches { + if !available.IsAvailable && enum.CacheStatusFromString(toServers[string(cache)].Status) == enum.CacheStatusReported { + count++ + } + } + return count } func createAPIPeerStates(peerStates map[enum.TrafficMonitorName]peer.Crstates, filter *PeerStateFilter, params url.Values) APIPeerStates { http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/10129f3f/traffic_monitor/experimental/traffic_monitor/manager/manager.go ---------------------------------------------------------------------- diff --git a/traffic_monitor/experimental/traffic_monitor/manager/manager.go b/traffic_monitor/experimental/traffic_monitor/manager/manager.go index 36e47c6..70af942 100644 --- a/traffic_monitor/experimental/traffic_monitor/manager/manager.go +++ b/traffic_monitor/experimental/traffic_monitor/manager/manager.go @@ -130,6 +130,7 @@ func Start(opsConfigFile string, cfg config.Config, staticAppData StaticAppData) errorCount, localCacheStatus, unpolledCaches, + monitorConfig, cfg, ) http://git-wip-us.apache.org/repos/asf/incubator-trafficcontrol/blob/10129f3f/traffic_monitor/experimental/traffic_monitor/manager/opsconfig.go ---------------------------------------------------------------------- diff --git a/traffic_monitor/experimental/traffic_monitor/manager/opsconfig.go b/traffic_monitor/experimental/traffic_monitor/manager/opsconfig.go index cdb78eb..2ced120 100644 --- a/traffic_monitor/experimental/traffic_monitor/manager/opsconfig.go +++ b/traffic_monitor/experimental/traffic_monitor/manager/opsconfig.go @@ -65,6 +65,7 @@ func StartOpsConfigManager( errorCount UintThreadsafe, localCacheStatus CacheAvailableStatusThreadsafe, unpolledCaches UnpolledCachesThreadsafe, + monitorConfig TrafficMonitorConfigMapThreadsafe, cfg config.Config, ) OpsConfigThreadsafe { @@ -125,6 +126,7 @@ func StartOpsConfigManager( localCacheStatus, lastStats, unpolledCaches, + monitorConfig, ) }, listenAddress, cfg.ServeReadTimeout, cfg.ServeWriteTimeout) if err != nil {
