Updated Branches: refs/heads/master a26de1996 -> 691be5c60
bug CS-15577: Added per gateway network usage for VPC and VPN usage. VPN usage uses 525 mark for outgoing traffic and 524 mark for incoming traffic status CS-15577: resolved fixed Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/691be5c6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/691be5c6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/691be5c6 Branch: refs/heads/master Commit: 691be5c60e40dee799e7267894572fa352060863 Parents: a26de19 Author: kishan <[email protected]> Authored: Fri Aug 17 17:07:13 2012 +0530 Committer: kishan <[email protected]> Committed: Fri Aug 17 17:07:13 2012 +0530 ---------------------------------------------------------------------- .../com/cloud/agent/api/NetworkUsageCommand.java | 42 ++- .../debian/config/opt/cloud/bin/ipsectunnel.sh | 4 +- .../debian/config/opt/cloud/bin/vpc_guestnw.sh | 17 - .../debian/config/opt/cloud/bin/vpc_netusage.sh | 93 ++++- .../hypervisor/vmware/resource/VmwareResource.java | 61 +++ .../xen/resource/XenServer56Resource.java | 50 ++ .../router/VirtualNetworkApplianceManagerImpl.java | 360 +++++++-------- .../VpcVirtualNetworkApplianceManagerImpl.java | 66 +++- 8 files changed, 461 insertions(+), 232 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/api/src/com/cloud/agent/api/NetworkUsageCommand.java ---------------------------------------------------------------------- diff --git a/api/src/com/cloud/agent/api/NetworkUsageCommand.java b/api/src/com/cloud/agent/api/NetworkUsageCommand.java index 21eac31..acb23cf 100644 --- a/api/src/com/cloud/agent/api/NetworkUsageCommand.java +++ b/api/src/com/cloud/agent/api/NetworkUsageCommand.java @@ -17,7 +17,6 @@ package com.cloud.agent.api; import com.cloud.agent.api.LogLevel.Log4jLevel; -import com.cloud.agent.api.to.NicTO; @LogLevel(Log4jLevel.Trace) public class NetworkUsageCommand extends Command { @@ -25,18 +24,20 @@ public class NetworkUsageCommand extends Command { private String domRName; private String option; boolean forVpc = false; - NicTO guestNic; + private String gatewayIP; + private String vpcCIDR; protected NetworkUsageCommand() { } - public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, NicTO guestNic) + public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, String gatewayIP) { this.privateIP = privateIP; this.domRName = domRName; this.forVpc = forVpc; - this.guestNic = guestNic; + this.gatewayIP = gatewayIP; + this.option = "get"; } public NetworkUsageCommand(String privateIP, String domRName, String option, boolean forVpc) @@ -47,6 +48,25 @@ public class NetworkUsageCommand extends Command { this.forVpc = forVpc; } + public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, String gatewayIP, String vpcCIDR) + { + this.privateIP = privateIP; + this.domRName = domRName; + this.forVpc = forVpc; + this.gatewayIP = gatewayIP; + this.option = "create"; + this.vpcCIDR = vpcCIDR; + } + + public NetworkUsageCommand(String privateIP, String domRName, String option, boolean forVpc, String gatewayIP) + { + this.privateIP = privateIP; + this.domRName = domRName; + this.forVpc = forVpc; + this.gatewayIP = gatewayIP; + this.option = option; + } + public String getPrivateIP() { return privateIP; } @@ -59,8 +79,20 @@ public class NetworkUsageCommand extends Command { return option; } + public boolean isForVpc() { + return forVpc; + } + + public String getVpcCIDR() { + return vpcCIDR; + } + + public String getGatewayIP() { + return gatewayIP; + } + @Override public boolean executeInSequence() { return false; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh ---------------------------------------------------------------------- diff --git a/patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh b/patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh index 1962212..0b8c992 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh +++ b/patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh @@ -54,9 +54,9 @@ start_ipsec() { enable_iptables_subnets() { for net in $rightnets do - sudo iptables -A FORWARD -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark + sudo iptables -I FORWARD -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark sudo iptables -A OUTPUT -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark - sudo iptables -A FORWARD -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark + sudo iptables -I FORWARD -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark sudo iptables -A INPUT -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark done return 0 http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh ---------------------------------------------------------------------- diff --git a/patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh b/patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh index 61c5a7e..0ca371c 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh +++ b/patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh @@ -97,21 +97,6 @@ desetup_dnsmasq() { sleep 1 } -setup_usage() { - sudo iptables -t mangle -N NETWORK_STATS_$dev - sudo iptables -t mangle -A NETWORK_STATS_$dev -s $subnet/$mask ! -d $vpccidr - sudo iptables -t mangle -A NETWORK_STATS_$dev -o $dev ! -s $vpccidr - sudo iptables -t mangle -A POSTROUTING -s $subnet/$mask -j NETWORK_STATS_$dev - sudo iptables -t mangle -A POSTROUTING -o $dev -j NETWORK_STATS_$dev -} - -desetup_usage() { - sudo iptables -t mangle -F NETWORK_STATS_$dev - sudo iptables -t mangle -D POSTROUTING -s $subnet/$mask -j NETWORK_STATS_$dev - sudo iptables -t mangle -D POSTROUTING -o $dev -j NETWORK_STATS_$dev - sudo iptables -t mangle -X NETWORK_STATS_$dev -} - create_guest_network() { logger -t cloud " $(basename $0): Create network on interface $dev, gateway $gw, network $ip/$mask " # setup ip configuration @@ -130,7 +115,6 @@ create_guest_network() { # set up hairpin sudo iptables -t nat -A POSTROUTING -s $subnet/$mask -o $dev -j SNAT --to-source $ip create_acl_chain - setup_usage setup_dnsmasq setup_apache2 } @@ -144,7 +128,6 @@ destroy_guest_network() { sudo iptables -t mangle -D PREROUTING -i $dev -m state --state ESTABLISHED,RELATED -j CONNMARK --restore-mark sudo iptables -t nat -A POSTROUTING -s $subnet/$mask -o $dev -j SNAT --to-source $ip destroy_acl_chain - desetup_usage desetup_dnsmasq desetup_apache2 } http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh ---------------------------------------------------------------------- diff --git a/patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh b/patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh index 54c6b55..743ad32 100755 --- a/patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh +++ b/patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh @@ -15,6 +15,8 @@ source /root/func.sh source /opt/cloud/bin/vpc_func.sh +vpnoutmark="0x525" +vpninmark="0x524" lock="biglock" locked=$(getLockFile $lock) if [ "$locked" != "1" ] @@ -23,25 +25,64 @@ then fi usage() { - printf "Usage: %s -[c|g|r] [-[a|d] <public interface>]\n" $(basename $0) >&2 + printf "Usage: %s -[c|g|r|n|d] [-l <public gateway>] [-v <vpc cidr>] \n" $(basename $0) >&2 } create_usage_rules () { - iptables-save|grep "NETWORK_STATS_$guestIp -i $ethDev" > /dev/null + iptables-save|grep "NETWORK_STATS_$ethDev" > /dev/null if [ $? -gt 0 ] then - iptables -A NETWORK_STATS_$guestIp -i $ethDev -s ! zcidr > /dev/null - fi - iptables-save|grep "NETWORK_STATS_$guestIp -o $ethDev" > /dev/null + iptables -N NETWORK_STATS_$ethDev > /dev/null; + iptables -I FORWARD -j NETWORK_STATS_$ethDev > /dev/null; + iptables -A NETWORK_STATS_$ethDev -o $ethDev -s $vcidr > /dev/null; + iptables -A NETWORK_STATS_$ethDev -i $ethDev -d $vcidr > /dev/null; + fi + return $? +} + +create_vpn_usage_rules () { + iptables-save|grep "VPN_STATS_$ethDev" > /dev/null if [ $? -gt 0 ] then - iptables -A NETWORK_STATS_$guestIp -o $ethDev -d ! zcidr > /dev/null + iptables -t mangle -N VPN_STATS_$ethDev > /dev/null; + iptables -t mangle -I FORWARD -j VPN_STATS_$ethDev > /dev/null; + iptables -t mangle -A VPN_STATS_$ethDev -o $ethDev -m mark --mark $vpnoutmark > /dev/null; + iptables -t mangle -A VPN_STATS_$ethDev -i $ethDev -m mark --mark $vpninmark > /dev/null; fi return $? } +remove_usage_rules () { + echo $ethDev >> /root/removedVifs + return $? +} + get_usage () { - iptables -t mangle -L NETWORK_STATS_$ethDev -n -v -x | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null + iptables -L NETWORK_STATS_$ethDev -n -v -x 2> /dev/null | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null + if [ -f /root/removedVifs ] + then + var=`cat /root/removedVifs` + # loop through vifs to be cleared + for i in $var; do + # Make sure vif doesn't exist + if [ ! -f /sys/class/net/$i ] + then + # flush rules and remove chain + iptables -F NETWORK_STATS_$i > /dev/null; + iptables -D FORWARD -j NETWORK_STATS_$i > /dev/null; + iptables -X NETWORK_STATS_$i > /dev/null; + iptables -t mangle -F VPN_STATS_$i > /dev/null; + iptables -t mangle -D FORWARD -j VPN_STATS_$i > /dev/null; + iptables -t mangle -X VPN_STATS_$i > /dev/null; + fi + done + rm /root/removedVifs + fi + return 0 +} + +get_vpn_usage () { + iptables -t mangle -L VPN_STATS_$ethDev -n -v -x | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null if [ $? -gt 0 ] then printf $? @@ -50,7 +91,7 @@ get_usage () { } reset_usage () { - iptables -t mangle -Z NETWORK_STATS_$ethDev > /dev/null + iptables -Z NETWORK_STATS_$ethDev > /dev/null if [ $? -gt 0 -a $? -ne 2 ] then return 1 @@ -63,9 +104,11 @@ cflag= gflag= rflag= lflag= +vflag= +nflag= +dflag= - -while getopts 'cgrl:' OPTION +while getopts 'cgndrl:v:' OPTION do case $OPTION in c) cflag=1 @@ -75,8 +118,15 @@ do r) rflag=1 ;; l) lflag=1 - guestIp="$OPTARG" + publicIp="$OPTARG" + ;; + v) vflag=1 + vcidr="$OPTARG" ;; + n) nflag=1 + ;; + d) dflag=1 + ;; i) #Do nothing, since it's parameter for host script ;; ?) usage @@ -85,18 +135,35 @@ do esac done +ethDev=$(getEthByIp $publicIp) if [ "$cflag" == "1" ] then - unlock_exit 0 $lock $locked + if [ "$ethDev" != "" ] + then + create_usage_rules + create_vpn_usage_rules + unlock_exit 0 $lock $locked + fi fi -ethDev=$(getEthByIp $guestIp) if [ "$gflag" == "1" ] then get_usage unlock_exit $? $lock $locked fi +if [ "$nflag" == "1" ] +then + get_vpn_usage + unlock_exit $? $lock $locked +fi + +if [ "$dflag" == "1" ] +then + remove_usage_rules + unlock_exit 0 $lock $locked +fi + if [ "$rflag" == "1" ] then reset_usage http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index c413fe9..838a132 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -491,6 +491,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } protected Answer execute(NetworkUsageCommand cmd) { + if ( cmd.isForVpc() ) { + return VPCNetworkUsage(cmd); + } if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource NetworkUsageCommand " + _gson.toJson(cmd)); } @@ -505,6 +508,64 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return answer; } + protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) { + String privateIp = cmd.getPrivateIP(); + String option = cmd.getOption(); + String publicIp = cmd.getGatewayIP(); + + + String args = "-l " + publicIp+ " "; + if (option.equals("get")) { + args += "-g"; + } else if (option.equals("create")) { + args += "-c"; + String vpcCIDR = cmd.getVpcCIDR(); + args += " -v " + vpcCIDR; + } else if (option.equals("reset")) { + args += "-r"; + } else if (option.equals("vpn")) { + args += "-n"; + } else if (option.equals("remove")) { + args += "-d"; + } else { + return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + } + try { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Executing /opt/cloud/bin/vpc_netusage.sh " + args + " on DomR " + privateIp); + } + VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); + + Pair<Boolean, String> resultPair = SshHelper.sshExecute(privateIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/vpc_netusage.sh " + args); + + if (!resultPair.first()) { + throw new Exception(" vpc network usage plugin call failed "); + } + + if (option.equals("get") || option.equals("vpn")) { + String result = resultPair.second(); + if (result == null || result.isEmpty()) { + throw new Exception(" vpc network usage get returns empty "); + } + long[] stats = new long[2]; + if (result != null) { + String[] splitResult = result.split(":"); + int i = 0; + while (i < splitResult.length - 1) { + stats[0] += (new Long(splitResult[i++])).longValue(); + stats[1] += (new Long(splitResult[i++])).longValue(); + } + return new NetworkUsageAnswer(cmd, "success", stats[0], stats[1]); + } + } + return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + } catch (Throwable e) { + s_logger.error("Unable to execute NetworkUsage command on DomR (" + privateIp + "), domR may not be ready yet. failure due to " + + VmwareHelper.getExceptionMessage(e), e); + } + return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + } + protected Answer execute(SetPortForwardingRulesCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource SetPortForwardingRulesCommand: " + _gson.toJson(cmd)); http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java index 4ad276f..9f672c5 100644 --- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java +++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServer56Resource.java @@ -160,7 +160,57 @@ public class XenServer56Resource extends CitrixResourceBase { return callHostPlugin(conn, "vmops", "routerProxy", "args", args); } + protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) { + try { + Connection conn = getConnection(); + String option = cmd.getOption(); + String publicIp = cmd.getGatewayIP(); + + String args = "vpc_netusage.sh " + cmd.getPrivateIP(); + args += " -l " + publicIp+ " "; + if (option.equals("get")) { + args += "-g"; + } else if (option.equals("create")) { + args += "-c"; + String vpcCIDR = cmd.getVpcCIDR(); + args += " -v " + vpcCIDR; + } else if (option.equals("reset")) { + args += "-r"; + } else if (option.equals("vpn")) { + args += "-n"; + } else if (option.equals("remove")) { + args += "-d"; + } else { + return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + } + + String result = callHostPlugin(conn, "vmops", "routerProxy", "args", args); + if (option.equals("get") || option.equals("vpn")) { + long[] stats = new long[2]; + if (result != null) { + String[] splitResult = result.split(":"); + int i = 0; + while (i < splitResult.length - 1) { + stats[0] += (new Long(splitResult[i++])).longValue(); + stats[1] += (new Long(splitResult[i++])).longValue(); + } + return new NetworkUsageAnswer(cmd, "success", stats[0], stats[1]); + } + } + if (result == null || result.isEmpty()) { + throw new Exception(" vpc network usage plugin call failed "); + } + return new NetworkUsageAnswer(cmd, "success", 0L, 0L); + } catch (Exception ex) { + s_logger.warn("Failed to get network usage stats due to ", ex); + return new NetworkUsageAnswer(cmd, ex); + } + } + protected NetworkUsageAnswer execute(NetworkUsageCommand cmd) { + if ( cmd.isForVpc() ) { + return VPCNetworkUsage(cmd); + } try { Connection conn = getConnection(); if(cmd.getOption()!=null && cmd.getOption().equals("create") ){ http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 8202511..e76588b 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -760,100 +760,96 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian @Override public void run() { - try{ - final List<DomainRouterVO> routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); - s_logger.debug("Found " + routers.size() + " running routers. "); - - for (DomainRouterVO router : routers) { - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long guestNtwkId : routerGuestNtwkIds) { - boolean forVpc = router.getVpcId() != null; - Network guestNtwk = _networkMgr.getNetwork(guestNtwkId); - Nic guestNic = _nicDao.findByInstanceIdAndNetworkId(guestNtwk.getId(), router.getId()); - NicProfile guestNicProfile = new NicProfile(guestNic, guestNtwk, guestNic.getBroadcastUri(), - guestNic.getIsolationUri(), _networkMgr.getNetworkRate(guestNtwk.getId(), router.getId()), - _networkMgr.isSecurityGroupSupportedInNetwork(guestNtwk), - _networkMgr.getNetworkTag(router.getHypervisorType(), guestNtwk)); - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, _itMgr.toNicTO(guestNicProfile, router.getHypervisorType())); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } - } - } catch (Exception e) { - s_logger.warn("Error while collecting network stats", e); - } + try{ + final List<DomainRouterVO> routers = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (DomainRouterVO router : routers) { + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + List<? extends Nic> routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Public) { + boolean forVpc = router.getVpcId() != null; + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), routerNic.getIp4Address(), router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if(previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } + } + } catch (Exception e) { + s_logger.warn("Error while collecting network stats", e); + } } } @@ -3294,95 +3290,93 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian public void prepareStop(VirtualMachineProfile<DomainRouterVO> profile){ //Collect network usage before stopping Vm VMInstanceVO vm = profile.getVirtualMachine(); - DomainRouterVO router = _routerDao.findById(vm.getId()); - if(router == null){ - return; - } - String privateIP = router.getPrivateIpAddress(); - - if (privateIP != null) { - List<Long> routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); - - for (Long guestNtwkId : routerGuestNtwkIds) { - boolean forVpc = router.getVpcId() != null; - Network guestNtwk = _networkMgr.getNetwork(guestNtwkId); - Nic guestNic = _nicDao.findByInstanceIdAndNetworkId(guestNtwk.getId(), router.getId()); - NicProfile guestNicProfile = new NicProfile(guestNic, guestNtwk, guestNic.getBroadcastUri(), - guestNic.getIsolationUri(), _networkMgr.getNetworkRate(guestNtwk.getId(), router.getId()), - _networkMgr.isSecurityGroupSupportedInNetwork(guestNtwk), - _networkMgr.getNetworkTag(router.getHypervisorType(), guestNtwk)); - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), - forVpc, _itMgr.toNicTO(guestNicProfile, router.getHypervisorType())); - UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), - router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (Exception e) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); - continue; - } - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - txn.start(); - UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), - router.getDataCenterIdToDeployIn(), guestNtwkId, null, router.getId(), router.getType().toString()); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - continue; - } - - if(previousStats != null - && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) - || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + - "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + - answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); - continue; - } - - if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesReceived() - + " Stored: " + stats.getCurrentBytesReceived()); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answer.getBytesReceived()); - if (stats.getCurrentBytesSent() > answer.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " + - "Assuming something went wrong and persisting it. Router: " + - answer.getRouterName()+" Reported: " + answer.getBytesSent() - + " Stored: " + stats.getCurrentBytesSent()); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answer.getBytesSent()); - _statsDao.update(stats.getId(), stats); - txn.commit(); - } catch (Exception e) { - txn.rollback(); - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() - + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); - } finally { - txn.close(); - } - } - } - } + + DomainRouterVO router = _routerDao.findById(vm.getId()); + if(router == null){ + return; + } + + String privateIP = router.getPrivateIpAddress(); + + if (privateIP != null) { + List<? extends Nic> routerNics = _nicDao.listByVmId(router.getId()); + for (Nic routerNic : routerNics) { + Network network = _networkMgr.getNetwork(routerNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Public) { + boolean forVpc = router.getVpcId() != null; + final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), + forVpc, routerNic.getIp4Address()); + UserStatisticsVO previousStats = _statsDao.findBy(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + NetworkUsageAnswer answer = null; + try { + answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); + } catch (Exception e) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId(), e); + continue; + } + + if (answer != null) { + if (!answer.getResult()) { + s_logger.warn("Error while collecting network stats from router: "+router.getInstanceName()+" from host: "+router.getHostId() + "; details: " + answer.getDetails()); + continue; + } + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + if ((answer.getBytesReceived() == 0) && (answer.getBytesSent() == 0)) { + s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); + continue; + } + txn.start(); + UserStatisticsVO stats = _statsDao.lock(router.getAccountId(), + router.getDataCenterIdToDeployIn(), network.getId(), null, router.getId(), router.getType().toString()); + if (stats == null) { + s_logger.warn("unable to find stats for account: " + router.getAccountId()); + continue; + } + + if(previousStats != null + && ((previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived()) + || (previousStats.getCurrentBytesSent() != stats.getCurrentBytesSent()))){ + s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + + "Ignoring current answer. Router: "+answer.getRouterName()+" Rcvd: " + + answer.getBytesReceived()+ "Sent: " +answer.getBytesSent()); + continue; + } + + if (stats.getCurrentBytesReceived() > answer.getBytesReceived()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesReceived() + + " Stored: " + stats.getCurrentBytesReceived()); + } + stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); + } + stats.setCurrentBytesReceived(answer.getBytesReceived()); + if (stats.getCurrentBytesSent() > answer.getBytesSent()) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Received # of bytes that's less than the last one. " + + "Assuming something went wrong and persisting it. Router: " + + answer.getRouterName()+" Reported: " + answer.getBytesSent() + + " Stored: " + stats.getCurrentBytesSent()); + } + stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); + } + stats.setCurrentBytesSent(answer.getBytesSent()); + _statsDao.update(stats.getId(), stats); + txn.commit(); + } catch (Exception e) { + txn.rollback(); + s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + + " Rx: " + answer.getBytesReceived() + "; Tx: " + answer.getBytesSent()); + } finally { + txn.close(); + } + } + } + } + } } @Override http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/691be5c6/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java index ce5ba34..4f3e02e 100644 --- a/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java @@ -29,6 +29,9 @@ import javax.ejb.Local; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager.OnError; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.PlugNicAnswer; import com.cloud.agent.api.PlugNicCommand; import com.cloud.agent.api.SetupGuestNetworkAnswer; @@ -96,6 +99,7 @@ import com.cloud.network.vpc.StaticRouteProfile; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcGateway; import com.cloud.network.vpc.VpcManager; +import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.PrivateIpDao; import com.cloud.network.vpc.dao.StaticRouteDao; import com.cloud.network.vpc.dao.VpcDao; @@ -103,6 +107,7 @@ import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.user.Account; +import com.cloud.user.UserStatisticsVO; import com.cloud.utils.Pair; import com.cloud.utils.component.Inject; import com.cloud.utils.db.DB; @@ -330,15 +335,13 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian PlugNicCommand plugNicCmd = new PlugNicCommand(nic, vm.getName()); Commands cmds = new Commands(OnError.Stop); - cmds.addCommand("plugnic", plugNicCmd); + cmds.addCommand("plugnic", plugNicCmd); _agentMgr.send(dest.getHost().getId(), cmds); - PlugNicAnswer plugNicAnswer = cmds.getAnswer(PlugNicAnswer.class); if (!(plugNicAnswer != null && plugNicAnswer.getResult())) { s_logger.warn("Unable to plug nic for vm " + vm.getHostName()); result = false; } - } catch (OperationTimedoutException e) { throw new AgentUnavailableException("Unable to plug nic for router " + vm.getHostName() + " in network " + network, dest.getHost().getId(), e); @@ -362,8 +365,12 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian if (router.getState() == State.Running) { try { + Commands cmds = new Commands(OnError.Stop); + if(network.getTrafficType() == TrafficType.Public){ + NetworkUsageCommand netUsageCmd = new NetworkUsageCommand(router.getPrivateIpAddress(), router.getInstanceName(), "remove", true, nic.getIp()); + cmds.addCommand(netUsageCmd); + } UnPlugNicCommand unplugNicCmd = new UnPlugNicCommand(nic, vm.getName()); - Commands cmds = new Commands(OnError.Stop); cmds.addCommand("unplugnic", unplugNicCmd); _agentMgr.send(dest.getHost().getId(), cmds); @@ -371,8 +378,14 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian if (!(unplugNicAnswer != null && unplugNicAnswer.getResult())) { s_logger.warn("Unable to unplug nic from router " + router); result = false; - } - + } else { + if(network.getTrafficType() == TrafficType.Public){ + NetworkUsageCommand netUsageCmd = new NetworkUsageCommand(router.getPrivateIpAddress(), router.getInstanceName(), "remove", true, nic.getIp()); + cmds = new Commands(OnError.Stop); + cmds.addCommand(netUsageCmd); + _agentMgr.send(dest.getHost().getId(), cmds); + } + } } catch (OperationTimedoutException e) { throw new AgentUnavailableException("Unable to unplug nic from rotuer " + router + " from network " + network, dest.getHost().getId(), e); @@ -565,7 +578,10 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian return false; } } - + + Commands netUsagecmds = new Commands(OnError.Continue); + VpcVO vpc = _vpcDao.findById(router.getVpcId()); + //2) Plug the nics for (String vlanTag : nicsToPlug.keySet()) { PublicIpAddress ip = nicsToPlug.get(vlanTag); @@ -600,6 +616,16 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian return false; } } + //Create network usage commands. Send commands to router after IPAssoc + NetworkUsageCommand netUsageCmd = new NetworkUsageCommand(router.getPrivateIpAddress(), router.getInstanceName(), true, defaultNic.getIp4Address(), vpc.getCidr()); + netUsagecmds.addCommand(netUsageCmd); + UserStatisticsVO stats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn(), + publicNtwk.getId(), publicNic.getIp4Address(), router.getId(), router.getType().toString()); + if (stats == null) { + stats = new UserStatisticsVO(router.getAccountId(), router.getDataCenterIdToDeployIn(), publicNic.getIp4Address(), router.getId(), + router.getType().toString(), publicNtwk.getId()); + _userStatsDao.persist(stats); + } } //3) apply the ips @@ -633,7 +659,10 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian return sendCommandsToRouter(router, cmds); } }); - + if(result && netUsagecmds.size() > 0){ + //After successful ipassoc, send commands to router + sendCommandsToRouter(router, netUsagecmds); + } return result; } @@ -764,6 +793,8 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian } } + List<Command> usageCmds = new ArrayList<Command>(); + //3) PREPARE PLUG NIC COMMANDS try { //add VPC router to public networks @@ -787,10 +818,18 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian _routerDao.update(routerVO.getId(), routerVO); } } - - PlugNicCommand plugNicCmd = new PlugNicCommand(getNicTO(router, publicNic.getNetworkId(), - publicNic.getBroadcastUri().toString()), router.getInstanceName()); + PlugNicCommand plugNicCmd = new PlugNicCommand(getNicTO(router, publicNic.getNetworkId(), publicNic.getBroadcastUri().toString()), router.getInstanceName()); cmds.addCommand(plugNicCmd); + VpcVO vpc = _vpcDao.findById(router.getVpcId()); + NetworkUsageCommand netUsageCmd = new NetworkUsageCommand(router.getPrivateIpAddress(), router.getInstanceName(), true, publicNic.getIp4Address(), vpc.getCidr()); + usageCmds.add(netUsageCmd); + UserStatisticsVO stats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterIdToDeployIn(), + publicNtwk.getId(), publicNic.getIp4Address(), router.getId(), router.getType().toString()); + if (stats == null) { + stats = new UserStatisticsVO(router.getAccountId(), router.getDataCenterIdToDeployIn(), publicNic.getIp4Address(), router.getId(), + router.getType().toString(), publicNtwk.getId()); + _userStatsDao.persist(stats); + } } // create ip assoc for source nat @@ -874,7 +913,10 @@ public class VpcVirtualNetworkApplianceManagerImpl extends VirtualNetworkApplian finalizeUserDataAndDhcpOnStart(cmds, router, provider, guestNic.getNetworkId()); } - + + //Add network usage commands + cmds.addCommands(usageCmds); + return true; }
