...and do not expect them to have load data available already. In this way, we know where to expect load data being available. This allows to continue balancing quickly while not acting blindly.
Signed-off-by: Klaus Aehlig <[email protected]> --- src/Ganeti/MaintD/Balance.hs | 71 +++++++++++++++++++++++++++++++++++++------- src/Ganeti/MaintD/Server.hs | 2 +- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/Ganeti/MaintD/Balance.hs b/src/Ganeti/MaintD/Balance.hs index 9c88ef4..365c1ce 100644 --- a/src/Ganeti/MaintD/Balance.hs +++ b/src/Ganeti/MaintD/Balance.hs @@ -40,11 +40,12 @@ module Ganeti.MaintD.Balance ) where import Control.Exception.Lifted (bracket) -import Control.Monad (liftM) +import Control.Monad (liftM, unless, when) import Control.Monad.IO.Class (liftIO) +import Data.IORef (IORef) import qualified Data.Set as Set import qualified Data.Map as Map -import Data.Maybe (mapMaybe) +import Data.Maybe (mapMaybe, isJust) import qualified Data.Traversable as Traversable import System.IO.Error (tryIOError) import Text.Printf (printf) @@ -66,6 +67,8 @@ import Ganeti.Jobs (submitJobs) import Ganeti.HTools.Types ( zeroUtil, DynUtil(cpuWeight), addUtil, subUtil , MoveJob) import Ganeti.Logging.Lifted (logDebug) +import Ganeti.MaintD.MemoryState ( MemoryState, getEvacuated + , addEvacuated, rmEvacuated) import Ganeti.MaintD.Utils (annotateOpCode) import qualified Ganeti.Luxi as L import Ganeti.OpCodes (MetaOpCode) @@ -138,9 +141,10 @@ findInstanceLoad _ _ = Nothing updateCPUInstance :: Node.List -> Container.Container AllReports -> Set.Set String + -> [ String ] -> Instance.Instance -> Result Instance.Instance -updateCPUInstance nl reports xeninsts inst = +updateCPUInstance nl reports xeninsts evacuated inst = let name = Instance.name inst nidx = Instance.pNode inst in if name `Set.member` xeninsts @@ -154,6 +158,8 @@ updateCPUInstance nl reports xeninsts inst = return $ inst { Instance.util = zeroUtil { cpuWeight = load } } _ | Node.offline $ Container.find nidx nl -> return $ inst { Instance.util = zeroUtil } + _ | Instance.name inst `elem` evacuated -> + return $ inst { Instance.util = zeroUtil } _ -> fail $ "Xen CPU data unavailable for " ++ name else let rep = rTotal $ Container.find nidx reports in case rep of MonD.CPUavgloadReport (CPUavgload _ _ ndload) -> @@ -170,9 +176,10 @@ updateCPUInstance nl reports xeninsts inst = updateCPULoad :: (Node.List, Instance.List) -> Container.Container AllReports -> Set.Set String + -> [ String ] -> Result (Node.List, Instance.List) -updateCPULoad (nl, il) reports xeninsts = do - il' <- Traversable.mapM (updateCPUInstance nl reports xeninsts) il +updateCPULoad (nl, il) reports xeninsts evacuated = do + il' <- Traversable.mapM (updateCPUInstance nl reports xeninsts evacuated) il let addNodeUtil n delta = n { Node.utilLoad = addUtil (Node.utilLoad n) delta , Node.utilLoadForth = addUtil (Node.utilLoadForth n) delta @@ -187,6 +194,26 @@ updateCPULoad (nl, il) reports xeninsts = do $ Container.find i il') nl $ Container.keys il return (nl', il') +-- | For an instance, given by name, verify if an individual load report is +-- available again. +cleanUpEvacuation :: IORef MemoryState + -> Instance.List + -> Container.Container AllReports + -> String + -> IO () +cleanUpEvacuation memstate il reports name = do + let insts = filter ((==) name . Instance.name) $ Container.elems il + case insts of + [] -> do + logDebug $ "Instnace " ++ name ++ "no longer on the cluster" + rmEvacuated memstate name + inst:_ -> do + let nidx = Instance.pNode inst + when (isJust . findInstanceLoad name + $ Container.find nidx reports) $ do + logDebug $ "Load data for " ++ name ++ " available again" + rmEvacuated memstate name + -- * Balancing -- | Transform an instance move into a submittable job. @@ -213,14 +240,25 @@ iterateBalance opts ini_tbl cmds = in iterateBalance opts next_tbl cmds' _ -> cmds +-- | List instances evacuated in a move job, if any. +evacuatedInsts :: (Node.List, Instance.List) + -> MoveJob + -> [String] +evacuatedInsts (nl, il) (_, idx, _, _) = + let inst = Container.find idx il + node = Container.find (Instance.pNode inst) nl + in [Instance.name inst | Node.offline node] + -- | Balance a single group, restricted to the allowed nodes and -- minimal gain. -balanceGroup :: L.Client +balanceGroup :: IORef MemoryState + -> Set.Set String + -> L.Client -> Set.Set Int -> Double -> (Int, (Node.List, Instance.List)) -> ResultT String IO [JobId] -balanceGroup client allowedNodes threshold (gidx, (nl, il)) = do +balanceGroup memstate xens client allowedNodes threshold (gidx, (nl, il)) = do logDebug $ printf "Balancing group %d, %d nodes, %d instances." gidx (Container.size nl) (Container.size il) let ini_cv = Metrics.compCV nl @@ -234,9 +272,14 @@ balanceGroup client allowedNodes threshold (gidx, (nl, il)) = do logDebug $ "First task group: " ++ show tasks now <- liftIO currentTimestamp let jobs = tasks >>= map (moveToJob now (nl, il)) + evacs = filter (`Set.member` xens) + (concat tasks >>= evacuatedInsts (nl, il)) if null jobs then return [] else do + unless (null evacs) $ do + logDebug $ "Evacuation of instances " ++ show evacs + liftIO $ addEvacuated memstate evacs jids <- liftIO $ submitJobs jobs client case jids of Bad e -> mkResultT . logAndBad @@ -248,16 +291,22 @@ balanceGroup client allowedNodes threshold (gidx, (nl, il)) = do -- | Carry out all the needed balancing, based on live CPU data, only touching -- the available nodes. Only carry out balancing steps where the gain is above -- the threshold. -balanceTask :: (Node.List, Instance.List) -- ^ current cluster configuration +balanceTask :: IORef MemoryState + -> (Node.List, Instance.List) -- ^ current cluster configuration -> Set.Set Int -- ^ node indices on which actions may be taken -> Double -- ^ threshold for improvement -> ResultT String IO [JobId] -- ^ jobs submitted -balanceTask (nl, il) okNodes threshold = do +balanceTask memstate (nl, il) okNodes threshold = do logDebug "Collecting dynamic load values" + evacuated <- getEvacuated memstate + logDebug $ "Not expecting load data from: " ++ show evacuated reports <- queryLoad nl xenInstances <- getXenInstances - (nl', il') <- mkResultT . return $ updateCPULoad (nl, il) reports xenInstances + (nl', il') <- mkResultT . return + $ updateCPULoad (nl, il) reports xenInstances evacuated + liftIO $ mapM_ (cleanUpEvacuation memstate il reports) evacuated let ngroups = ClusterUtils.splitCluster nl' il' luxiSocket <- liftIO Path.defaultQuerySocket bracket (liftIO $ L.getLuxiClient luxiSocket) (liftIO . L.closeClient) $ \c -> - liftM concat $ mapM (balanceGroup c okNodes threshold) ngroups + liftM concat $ mapM (balanceGroup memstate xenInstances c okNodes threshold) + ngroups diff --git a/src/Ganeti/MaintD/Server.hs b/src/Ganeti/MaintD/Server.hs index b220b2d..8681496 100644 --- a/src/Ganeti/MaintD/Server.hs +++ b/src/Ganeti/MaintD/Server.hs @@ -144,7 +144,7 @@ maintenance memstate = do (bal, thresh) <- withErrorT show $ runNewWConfdClient maintenanceBalancing when (bal && not (Set.null nidxs')) $ do logDebug $ "Will balance unaffected nodes, threshold " ++ show thresh - jobs' <- balanceTask (nl, il) nidxs thresh + jobs' <- balanceTask memstate (nl, il) nidxs thresh logDebug $ "Balancing jobs submitted: " ++ show (map fromJobId jobs') unless (null jobs') . liftIO $ appendJobs memstate jobs' -- 2.5.0.rc2.392.g76e840b
