LGTM
On Tue, May 27, 2014 at 10:24 AM, 'Klaus Aehlig' via ganeti-devel < [email protected]> wrote: > > > commit c22f29fc38ef08225428bbc72f7b99ac25d7166f > Merge: 1d425eb c3a9cd3 > Author: Klaus Aehlig <[email protected]> > Date: Tue May 27 09:51:17 2014 +0200 > > Merge branch 'stable-2.11' into stable-2.12 > > * stable-2.11 > (no changes) > > * stable-2.10 > Use more efficient statistics for the standard deviation > Use statistics updates when allocating on pairs > Factor score computation through abstract statistics > Verify the update of the standard deviation statistics > Add data type for abstract statistics > Relax test requirements > Fix gnt-network client wrt instances report > Fix QueryNetworks wrt instances > > Conflicts: > src/Ganeti/HTools/Cluster.hs: manually apply > patch b366d7323e7 to stable-2.11 version > src/Ganeti/Utils.hs: take all addition > > Signed-off-by: Klaus Aehlig <[email protected]> > > diff --cc src/Ganeti/HTools/Cluster.hs > index 0e585d7,04f770a..94d9a9f > --- a/src/Ganeti/HTools/Cluster.hs > +++ b/src/Ganeti/HTools/Cluster.hs > @@@ -340,49 -346,80 +346,80 @@@ detailedCVInfo = map fst detailedCVInfo > detailedCVWeights :: [Double] > detailedCVWeights = map fst detailedCVInfo > > - -- | Compute the mem and disk covariance. > - compDetailedCV :: [Node.Node] -> [Double] > - compDetailedCV all_nodes = > + -- | The aggregation functions for the weights > + detailedCVAggregation :: [([Double] -> Statistics, Bool)] > + detailedCVAggregation = map snd detailedCVInfoExt > + > + -- | The bit vector describing which parts of the statistics are > + -- for online nodes. > + detailedCVOnlineStatus :: [Bool] > + detailedCVOnlineStatus = map snd detailedCVAggregation > + > + -- | Compute statistical measures of a single node. > + compDetailedCVNode :: Node.Node -> [Double] > + compDetailedCVNode node = > + let mem = Node.pMem node > + dsk = Node.pDsk node > + n1 = fromIntegral > + $ if Node.failN1 node > + then length (Node.sList node) + length (Node.pList node) > + else 0 > + res = Node.pRem node > + ipri = fromIntegral . length $ Node.pList node > + isec = fromIntegral . length $ Node.sList node > + ioff = ipri + isec > - cpu = Node.pCpu node > ++ cpu = Node.pCpuEff node > + DynUtil c1 m1 d1 nn1 = Node.utilLoad node > + DynUtil c2 m2 d2 nn2 = Node.utilPool node > + (c_load, m_load, d_load, n_load) = (c1/c2, m1/m2, d1/d2, nn1/nn2) > + pri_tags = fromIntegral $ Node.conflictingPrimaries node > + spindles = Node.instSpindles node / Node.hiSpindles node > + in [ mem, dsk, n1, res, ioff, ipri, cpu > + , c_load, m_load, d_load, n_load > + , pri_tags, spindles > + ] > + > + -- | Compute the statistics of a cluster. > + compClusterStatistics :: [Node.Node] -> [Statistics] > + compClusterStatistics all_nodes = > let (offline, nodes) = partition Node.offline all_nodes > - mem_l = map Node.pMem nodes > - dsk_l = map Node.pDsk nodes > - -- metric: memory covariance > - mem_cv = stdDev mem_l > - -- metric: disk covariance > - dsk_cv = stdDev dsk_l > - -- metric: count of instances living on N1 failing nodes > - n1_score = fromIntegral . sum . map (\n -> length (Node.sList n) + > - length (Node.pList n)) . > - filter Node.failN1 $ nodes :: Double > - res_l = map Node.pRem nodes > - -- metric: reserved memory covariance > - res_cv = stdDev res_l > - -- offline instances metrics > - offline_ipri = sum . map (length . Node.pList) $ offline > - offline_isec = sum . map (length . Node.sList) $ offline > - -- metric: count of instances on offline nodes > - off_score = fromIntegral (offline_ipri + offline_isec)::Double > - -- metric: count of primary instances on offline nodes (this > - -- helps with evacuation/failover of primary instances on > - -- 2-node clusters with one node offline) > - off_pri_score = fromIntegral offline_ipri::Double > - cpu_l = map Node.pCpuEff nodes > - -- metric: covariance of effective vcpu/pcpu ratio > - cpu_cv = stdDev cpu_l > - -- metrics: covariance of cpu, memory, disk and network load > - (c_load, m_load, d_load, n_load) = > - unzip4 $ map (\n -> > - let DynUtil c1 m1 d1 n1 = Node.utilLoad n > - DynUtil c2 m2 d2 n2 = Node.utilPool n > - in (c1/c2, m1/m2, d1/d2, n1/n2)) nodes > - -- metric: conflicting instance count > - pri_tags_inst = sum $ map Node.conflictingPrimaries nodes > - pri_tags_score = fromIntegral pri_tags_inst::Double > - -- metric: spindles % > - spindles_cv = map (\n -> Node.instSpindles n / Node.hiSpindles n) > nodes > - in [ mem_cv, dsk_cv, n1_score, res_cv, off_score, off_pri_score, cpu_cv > - , stdDev c_load, stdDev m_load , stdDev d_load, stdDev n_load > - , pri_tags_score, stdDev spindles_cv ] > + offline_values = transpose (map compDetailedCVNode offline) > + ++ repeat [] > + -- transpose of an empty list is empty and not k times the empty > list, as > + -- would be the transpose of a 0 x k matrix > + online_values = transpose $ map compDetailedCVNode nodes > + aggregate (f, True) (onNodes, _) = f onNodes > + aggregate (f, False) (_, offNodes) = f offNodes > + in zipWith aggregate detailedCVAggregation > + $ zip online_values offline_values > + > + -- | Update a cluster statistics by replacing the contribution of one > + -- node by that of another. > + updateClusterStatistics :: [Statistics] > + -> (Node.Node, Node.Node) -> [Statistics] > + updateClusterStatistics stats (old, new) = > + let update = zip (compDetailedCVNode old) (compDetailedCVNode new) > + online = not $ Node.offline old > + updateStat forOnline stat upd = if forOnline == online > + then updateStatistics stat upd > + else stat > + in zipWith3 updateStat detailedCVOnlineStatus stats update > + > + -- | Update a cluster statistics twice. > + updateClusterStatisticsTwice :: [Statistics] > + -> (Node.Node, Node.Node) > + -> (Node.Node, Node.Node) > + -> [Statistics] > + updateClusterStatisticsTwice s a = > + updateClusterStatistics (updateClusterStatistics s a) > + > + -- | Compute cluster statistics > + compDetailedCV :: [Node.Node] -> [Double] > + compDetailedCV = map getStatisticValue . compClusterStatistics > + > + -- | Compute the cluster score from its statistics > + compCVfromStats :: [Statistics] -> Double > + compCVfromStats = sum . zipWith (*) detailedCVWeights . map > getStatisticValue > > -- | Compute the /total/ variance. > compCVNodes :: [Node.Node] -> Double > diff --cc src/Ganeti/Utils.hs > index 91d8148,e8cac42..d5eeb76 > --- a/src/Ganeti/Utils.hs > +++ b/src/Ganeti/Utils.hs > @@@ -1,4 -1,4 +1,4 @@@ > - {-# LANGUAGE FlexibleContexts #-} > -{-# LANGUAGE BangPatterns #-} > ++{-# LANGUAGE FlexibleContexts, BangPatterns #-} > > {-| Utility functions. -} > > @@@ -28,7 -28,11 +28,12 @@@ module Ganeti.Util > , debugFn > , debugXy > , sepSplit > + , findFirst > + , Statistics > + , getSumStatistics > + , getStdDevStatistics > + , getStatisticValue > + , updateStatistics > , stdDev > , if' > , select > > -- > Klaus Aehlig > Google Germany GmbH, Dienerstr. 12, 80331 Muenchen > Registergericht und -nummer: Hamburg, HRB 86891 > Sitz der Gesellschaft: Hamburg > Geschaeftsfuehrer: Graham Law, Christine Elizabeth Flores >
