commit c22f29fc38ef08225428bbc72f7b99ac25d7166f
Merge: 1d425eb c3a9cd3
Author: Klaus Aehlig <[email protected]>
Date: Tue May 27 09:51:17 2014 +0200
Merge branch 'stable-2.11' into stable-2.12
* stable-2.11
(no changes)
* stable-2.10
Use more efficient statistics for the standard deviation
Use statistics updates when allocating on pairs
Factor score computation through abstract statistics
Verify the update of the standard deviation statistics
Add data type for abstract statistics
Relax test requirements
Fix gnt-network client wrt instances report
Fix QueryNetworks wrt instances
Conflicts:
src/Ganeti/HTools/Cluster.hs: manually apply
patch b366d7323e7 to stable-2.11 version
src/Ganeti/Utils.hs: take all addition
Signed-off-by: Klaus Aehlig <[email protected]>
diff --cc src/Ganeti/HTools/Cluster.hs
index 0e585d7,04f770a..94d9a9f
--- a/src/Ganeti/HTools/Cluster.hs
+++ b/src/Ganeti/HTools/Cluster.hs
@@@ -340,49 -346,80 +346,80 @@@ detailedCVInfo = map fst detailedCVInfo
detailedCVWeights :: [Double]
detailedCVWeights = map fst detailedCVInfo
- -- | Compute the mem and disk covariance.
- compDetailedCV :: [Node.Node] -> [Double]
- compDetailedCV all_nodes =
+ -- | The aggregation functions for the weights
+ detailedCVAggregation :: [([Double] -> Statistics, Bool)]
+ detailedCVAggregation = map snd detailedCVInfoExt
+
+ -- | The bit vector describing which parts of the statistics are
+ -- for online nodes.
+ detailedCVOnlineStatus :: [Bool]
+ detailedCVOnlineStatus = map snd detailedCVAggregation
+
+ -- | Compute statistical measures of a single node.
+ compDetailedCVNode :: Node.Node -> [Double]
+ compDetailedCVNode node =
+ let mem = Node.pMem node
+ dsk = Node.pDsk node
+ n1 = fromIntegral
+ $ if Node.failN1 node
+ then length (Node.sList node) + length (Node.pList node)
+ else 0
+ res = Node.pRem node
+ ipri = fromIntegral . length $ Node.pList node
+ isec = fromIntegral . length $ Node.sList node
+ ioff = ipri + isec
- cpu = Node.pCpu node
++ cpu = Node.pCpuEff node
+ DynUtil c1 m1 d1 nn1 = Node.utilLoad node
+ DynUtil c2 m2 d2 nn2 = Node.utilPool node
+ (c_load, m_load, d_load, n_load) = (c1/c2, m1/m2, d1/d2, nn1/nn2)
+ pri_tags = fromIntegral $ Node.conflictingPrimaries node
+ spindles = Node.instSpindles node / Node.hiSpindles node
+ in [ mem, dsk, n1, res, ioff, ipri, cpu
+ , c_load, m_load, d_load, n_load
+ , pri_tags, spindles
+ ]
+
+ -- | Compute the statistics of a cluster.
+ compClusterStatistics :: [Node.Node] -> [Statistics]
+ compClusterStatistics all_nodes =
let (offline, nodes) = partition Node.offline all_nodes
- mem_l = map Node.pMem nodes
- dsk_l = map Node.pDsk nodes
- -- metric: memory covariance
- mem_cv = stdDev mem_l
- -- metric: disk covariance
- dsk_cv = stdDev dsk_l
- -- metric: count of instances living on N1 failing nodes
- n1_score = fromIntegral . sum . map (\n -> length (Node.sList n) +
- length (Node.pList n)) .
- filter Node.failN1 $ nodes :: Double
- res_l = map Node.pRem nodes
- -- metric: reserved memory covariance
- res_cv = stdDev res_l
- -- offline instances metrics
- offline_ipri = sum . map (length . Node.pList) $ offline
- offline_isec = sum . map (length . Node.sList) $ offline
- -- metric: count of instances on offline nodes
- off_score = fromIntegral (offline_ipri + offline_isec)::Double
- -- metric: count of primary instances on offline nodes (this
- -- helps with evacuation/failover of primary instances on
- -- 2-node clusters with one node offline)
- off_pri_score = fromIntegral offline_ipri::Double
- cpu_l = map Node.pCpuEff nodes
- -- metric: covariance of effective vcpu/pcpu ratio
- cpu_cv = stdDev cpu_l
- -- metrics: covariance of cpu, memory, disk and network load
- (c_load, m_load, d_load, n_load) =
- unzip4 $ map (\n ->
- let DynUtil c1 m1 d1 n1 = Node.utilLoad n
- DynUtil c2 m2 d2 n2 = Node.utilPool n
- in (c1/c2, m1/m2, d1/d2, n1/n2)) nodes
- -- metric: conflicting instance count
- pri_tags_inst = sum $ map Node.conflictingPrimaries nodes
- pri_tags_score = fromIntegral pri_tags_inst::Double
- -- metric: spindles %
- spindles_cv = map (\n -> Node.instSpindles n / Node.hiSpindles n) nodes
- in [ mem_cv, dsk_cv, n1_score, res_cv, off_score, off_pri_score, cpu_cv
- , stdDev c_load, stdDev m_load , stdDev d_load, stdDev n_load
- , pri_tags_score, stdDev spindles_cv ]
+ offline_values = transpose (map compDetailedCVNode offline)
+ ++ repeat []
+ -- transpose of an empty list is empty and not k times the empty list,
as
+ -- would be the transpose of a 0 x k matrix
+ online_values = transpose $ map compDetailedCVNode nodes
+ aggregate (f, True) (onNodes, _) = f onNodes
+ aggregate (f, False) (_, offNodes) = f offNodes
+ in zipWith aggregate detailedCVAggregation
+ $ zip online_values offline_values
+
+ -- | Update a cluster statistics by replacing the contribution of one
+ -- node by that of another.
+ updateClusterStatistics :: [Statistics]
+ -> (Node.Node, Node.Node) -> [Statistics]
+ updateClusterStatistics stats (old, new) =
+ let update = zip (compDetailedCVNode old) (compDetailedCVNode new)
+ online = not $ Node.offline old
+ updateStat forOnline stat upd = if forOnline == online
+ then updateStatistics stat upd
+ else stat
+ in zipWith3 updateStat detailedCVOnlineStatus stats update
+
+ -- | Update a cluster statistics twice.
+ updateClusterStatisticsTwice :: [Statistics]
+ -> (Node.Node, Node.Node)
+ -> (Node.Node, Node.Node)
+ -> [Statistics]
+ updateClusterStatisticsTwice s a =
+ updateClusterStatistics (updateClusterStatistics s a)
+
+ -- | Compute cluster statistics
+ compDetailedCV :: [Node.Node] -> [Double]
+ compDetailedCV = map getStatisticValue . compClusterStatistics
+
+ -- | Compute the cluster score from its statistics
+ compCVfromStats :: [Statistics] -> Double
+ compCVfromStats = sum . zipWith (*) detailedCVWeights . map getStatisticValue
-- | Compute the /total/ variance.
compCVNodes :: [Node.Node] -> Double
diff --cc src/Ganeti/Utils.hs
index 91d8148,e8cac42..d5eeb76
--- a/src/Ganeti/Utils.hs
+++ b/src/Ganeti/Utils.hs
@@@ -1,4 -1,4 +1,4 @@@
- {-# LANGUAGE FlexibleContexts #-}
-{-# LANGUAGE BangPatterns #-}
++{-# LANGUAGE FlexibleContexts, BangPatterns #-}
{-| Utility functions. -}
@@@ -28,7 -28,11 +28,12 @@@ module Ganeti.Util
, debugFn
, debugXy
, sepSplit
+ , findFirst
+ , Statistics
+ , getSumStatistics
+ , getStdDevStatistics
+ , getStatisticValue
+ , updateStatistics
, stdDev
, if'
, select
--
Klaus Aehlig
Google Germany GmbH, Dienerstr. 12, 80331 Muenchen
Registergericht und -nummer: Hamburg, HRB 86891
Sitz der Gesellschaft: Hamburg
Geschaeftsfuehrer: Graham Law, Christine Elizabeth Flores
