Repository: qpid-dispatch Updated Branches: refs/heads/config-read-write f163d38ee -> 91df5a675
DISPATCH-834 Added ability to deploy to multiple machines Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/91df5a67 Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/91df5a67 Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/91df5a67 Branch: refs/heads/config-read-write Commit: 91df5a6754b1c5778cf758c47bb0b4d271f9734d Parents: f163d38 Author: Ernest Allen <eal...@redhat.com> Authored: Wed Oct 11 08:30:39 2017 -0400 Committer: Ernest Allen <eal...@redhat.com> Committed: Wed Oct 11 08:30:39 2017 -0400 ---------------------------------------------------------------------- console/config/config.py | 189 ++++++++++++++++++++++-------- console/config/css/mock.css | 10 ++ console/config/html/qdrTopology.html | 30 ++++- console/config/js/qdrTopology.js | 151 ++++++++++++++++++++---- console/config/mock/section.py | 5 + 5 files changed, 311 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/config.py ---------------------------------------------------------------------- diff --git a/console/config/config.py b/console/config/config.py index 193b4ad..d17bc34 100755 --- a/console/config/config.py +++ b/console/config/config.py @@ -29,8 +29,12 @@ import SimpleHTTPServer import SocketServer import json import cStringIO +import yaml +import threading +import subprocess import pdb +from distutils.spawn import find_executable def id_generator(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) @@ -38,7 +42,7 @@ def id_generator(size=6, chars=string.ascii_uppercase + string.digits): get_class = lambda x: globals()[x] sectionKeys = {"log": "module", "sslProfile": "name", "connector": "port", "listener": "port"} -# borrowed from qpid-dispatch/python/qpid_dispatch_internal/management/config.py +# modified from qpid-dispatch/python/qpid_dispatch_internal/management/config.py def _parse(lines): """Parse config file format into a section list""" begin = re.compile(r'([\w-]+)[ \t]*{') # WORD { @@ -50,7 +54,10 @@ def _parse(lines): """Do substitutions to make line json-friendly""" line = line.strip() if line.startswith("#"): - return "" + if line.startswith("#deploy_host:"): + line = line[1:] + else: + return "" # 'pattern:' is a special snowflake. It allows '#' characters in # its value, so they cannot be treated as comment delimiters if line.split(':')[0].strip().lower() == "pattern": @@ -92,7 +99,9 @@ class Manager(object): def __init__(self, topology, verbose): self.topology = topology self.verbose = verbose - self.base = "topologies/" + self.topo_base = "topologies/" + self.deploy_base = "deployments/" + self.state = None def operation(self, op, request): m = op.replace("-", "_") @@ -105,6 +114,85 @@ class Manager(object): print "Got request " + op return method(request) + def ANSIBLE_INSTALLED(self, request): + if self.verbose: + print "Ansible is", "installed" if find_executable("ansible") else "not installed" + return "installed" if find_executable("ansible") else "" + + # if the node has listeners, and one of them has an http:'true' + def has_console(self, node): + #n = False + #return node.get('listeners') and any([n or h.get('http') for l, h in node.get('listeners').iteritems()]) + + listeners = node.get('listeners') + if listeners: + for k, listener in listeners.iteritems(): + if listener.get('http'): + return True + + return False + + def DEPLOY(self, request): + nodes = request["nodes"] + topology = request["topology"] + + self.PUBLISH(request, deploy=True) + + inventory = {'deploy_routers': + {'vars': {'topology': topology}, + 'hosts': {} + } + } + hosts = inventory['deploy_routers']['hosts'] + + #pdb.set_trace() + for node in nodes: + if node['cls'] == 'router': + host = node['host'] + if not host in hosts: + hosts[host] = {'nodes': [], 'create_console': False} + # if any of the nodes for this host has a console, set create_console for this host to true + hosts[host]['create_console'] = (hosts[host]['create_console'] or self.has_console(node)) + hosts[host]['nodes'].append(node['name']) + # local hosts need to be marked as such + if host in ('0.0.0.0', 'localhost', '127.0.0.1'): + hosts[host]['ansible_connection'] = 'local' + + with open(self.deploy_base + 'inventory.yml', 'w') as n: + yaml.safe_dump(inventory, n, default_flow_style=False) + + # start ansible-playbook in separate thread and callback when done + def popenCallback(callback, args): + def popen(callback, args): + # send all output to deploy.txt so we can send it to the console in DEPLOY_STATUS + with open('deploy.txt', 'w') as fout: + proc = subprocess.Popen(args, stdout=fout, stderr=fout) + proc.wait() + callback() + return + thread = threading.Thread(target=popen, args=(callback, args)) + thread.start() + + def ansible_done(): + if self.verbose: + print "-------------- DEPLOYMENT DONE ----------------" + self.state = "DONE" + + self.state = "DEPLOYING" + popenCallback(ansible_done, ['ansible-playbook', self.deploy_base + 'install_dispatch.yaml', '-i', self.deploy_base + 'inventory.yml']) + + return "deployment started" + + def DEPLOY_STATUS(self, request): + with open('deploy.txt', 'r') as fin: + content = fin.readlines() + + # remove leading blank line + if len(content) > 1 and content[0] == '\n': + content.pop(0) + + return [''.join(content), self.state] + def GET_LOG(self, request): return [] @@ -118,7 +206,7 @@ class Manager(object): nodes = [] links = [] - dc = DirectoryConfigs('./' + self.base + topology + '/') + dc = DirectoryConfigs('./' + self.topo_base + topology + '/') configs = dc.configs port_map = [] @@ -126,6 +214,8 @@ class Manager(object): port_map.append({'connectors': [], 'listeners': []}) node = {} for sect in configs[file]: + # remove notes to self + host = sect[1].pop('deploy_host', None) section = dc.asSection(sect) if section: if section.type == "router": @@ -133,15 +223,11 @@ class Manager(object): node["nodeType"] = unicode("inter-router") node["name"] = section.entries["id"] node["key"] = "amqp:/_topo/0/" + node["name"] + "/$management" + if host: + node['host'] = host nodes.append(node) elif section.type in sectionKeys: - # look for a host in a listener - if section.type == 'listener': - host = section.entries.get('host') - if host and 'host' not in node: - node['host'] = host - role = section.entries.get('role') if role == 'inter-router': # we are processing an inter-router listener or connector: so create a link @@ -171,50 +257,20 @@ class Manager(object): return unicode(self.topology) def GET_TOPOLOGY_LIST(self, request): - return [unicode(f) for f in os.listdir(self.base) if os.path.isdir(self.base + f)] + return [unicode(f) for f in os.listdir(self.topo_base) if os.path.isdir(self.topo_base + f)] def SWITCH(self, request): self.topology = request["topology"] - tdir = './' + self.base + self.topology + '/' + tdir = './' + self.topo_base + self.topology + '/' if not os.path.exists(tdir): os.makedirs(tdir) return self.LOAD(request) - def FIND_DIR(self, request): - dir = request['relativeDir'] - files = request['fileList'] - # find a directory with this name that contains these files - - def SHOW_CONFIG(self, request): nodeIndex = request['nodeIndex'] return self.PUBLISH(request, nodeIndex) - def PUBLISH(self, request, nodeIndex=None): - nodes = request["nodes"] - links = request["links"] - topology = request["topology"] - settings = request["settings"] - http_port = settings.get('http_port', 5675) - listen_port = settings.get('internal_port', 2000) - default_host = settings.get('default_host', '0.0.0.0') - - if nodeIndex and nodeIndex >= len(nodes): - return "Node index out of range" - - if self.verbose: - if nodeIndex is None: - print("PUBLISHing to " + topology) - else: - print("Creating config for " + topology + " node " + nodes[nodeIndex]['name']) - - if nodeIndex is None: - # remove all .conf files from the output dir. they will be recreated below possibly under new names - for f in glob(self.base + topology + "/*.conf"): - if self.verbose: - print "Removing", f - os.remove(f) - + def _connect_(self, links, nodes, default_host, listen_port): for link in links: s = nodes[link['source']] t = nodes[link['target']] @@ -239,6 +295,36 @@ class Manager(object): t['conns'].append({"port": lport, "host": lhost}) t['conn_to'].append(s['name']) + def PUBLISH(self, request, nodeIndex=None, deploy=False): + nodes = request["nodes"] + links = request["links"] + topology = request["topology"] + settings = request["settings"] + http_port = settings.get('http_port', 5675) + listen_port = settings.get('internal_port', 2000) + default_host = settings.get('default_host', '0.0.0.0') + + if nodeIndex and nodeIndex >= len(nodes): + return "Node index out of range" + + if self.verbose: + if nodeIndex is not None: + print("Creating config for " + topology + " node " + nodes[nodeIndex]['name']) + elif deploy: + print("DEPLOYing to " + topology) + else: + print("PUBLISHing to " + topology) + + if nodeIndex is None: + # remove all .conf files from the output dir. they will be recreated below possibly under new names + for f in glob(self.topo_base + topology + "/*.conf"): + if self.verbose: + print "Removing", f + os.remove(f) + + # establish connections and listeners for each node based on links + self._connect_(links, nodes, default_host, listen_port) + # now process all the routers for node in nodes: if node['nodeType'] == 'inter-router': @@ -246,10 +332,10 @@ class Manager(object): print "------------- processing node", node["name"], "---------------" nname = node["name"] - if nodeIndex is None: - config_fp = open(self.base + topology + "/" + nname + ".conf", "w+") - else: + if nodeIndex is not None: config_fp = cStringIO.StringIO() + else: + config_fp = open(self.topo_base + topology + "/" + nname + ".conf", "w+") # add a router section in the config file r = RouterSection(**node) @@ -258,6 +344,8 @@ class Manager(object): else: r.setEntry('mode', 'interior') r.setEntry('id', node['name']) + if nodeIndex is None: + r.setEntry('deploy_host', node.get('host', '')) config_fp.write(str(r) + "\n") # write other sections @@ -269,10 +357,15 @@ class Manager(object): c = get_class(cname) if sectionKey == "listener" and o['port'] != 'amqp' and int(o['port']) == http_port: config_fp.write("\n# Listener for a console\n") + if deploy: + o['httpRoot'] = '/usr/local/share/qpid-dispatch/stand-alone' + if node.get('host') == o.get('host'): + o['host'] = '0.0.0.0' config_fp.write(str(c(**o)) + "\n") if 'listener' in node: - lhost = node.get('host', default_host) + # always listen on localhost + lhost = "0.0.0.0" listenerSection = ListenerSection(node['listener'], **{'host': lhost, 'role': 'inter-router'}) if 'listen_from' in node and len(node['listen_from']) > 0: config_fp.write("\n# listener for connectors from " + ', '.join(node['listen_from']) + "\n") @@ -282,6 +375,8 @@ class Manager(object): for idx, conns in enumerate(node['conns']): conn_port = conns['port'] conn_host = conns['host'] + if node.get('host') == conn_host: + conn_host = "0.0.0.0" connectorSection = ConnectorSection(conn_port, **{'host': conn_host, 'role': 'inter-router'}) if 'conn_to' in node and len(node['conn_to']) > idx: config_fp.write("\n# connect to " + node['conn_to'][idx] + "\n") http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/css/mock.css ---------------------------------------------------------------------- diff --git a/console/config/css/mock.css b/console/config/css/mock.css index 307ee8c..6ae1156 100644 --- a/console/config/css/mock.css +++ b/console/config/css/mock.css @@ -125,4 +125,14 @@ div.boolean label { .alert { max-width: 20em; +} + +input.router-host { + width: 45em; +} + +pre.tail { + max-height: 15em; + min-height: 15em; + overflow-y: scroll; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/html/qdrTopology.html ---------------------------------------------------------------------- diff --git a/console/config/html/qdrTopology.html b/console/config/html/qdrTopology.html index 5dc9d5b..7b6f794 100644 --- a/console/config/html/qdrTopology.html +++ b/console/config/html/qdrTopology.html @@ -20,13 +20,14 @@ under the License. <div id="buttonBar" class="navbar-primary"> Current topology <select ng-model="mockTopologyDir" ng-options="item for item in mockTopologies"></select> - <button class="btn btn-primary" type="button" ng-click="Publish()">Publish</button> - <button class="btn btn-primary" type="button" ng-click="Clear()">Clear</button> - <button class="btn btn-primary pull-right" type="button" ng-click="doSettings()">Settings</button> - <button class="btn btn-primary pull-right" type="button" ng-click="showNewDlg()">New Topology</button> + <button class="btn btn-primary" type="button" ng-click="Publish()" title="Save this topology">Publish</button> + <button class="btn btn-primary" type="button" ng-click="Clear()" title="Remove all routers">Clear</button> + <button class="btn btn-primary" type="button" ng-click="Deploy()" ng-disabled="!canDeploy()" ng-if="ansible" title="Deploy this topology">Deploy</button> + <button class="btn btn-primary pull-right" type="button" ng-click="doSettings()" title="Show global settings">Settings</button> + <button class="btn btn-primary pull-right" type="button" ng-click="showNewDlg()" title="Enter a new topology name">New Topology</button> <div class="selected-node pull-right"> <button class="btn btn-primary" type="button" ng-click="addAnotherNode(true)"><b class="plus caret"></b> Add new router</button> - <button id="action_button" class="btn btn-primary" type="button" ng-disabled="!selected_node" ng-click="showActions($event)">Actions <b class="down caret"></b></button> on selected router + <button id="action_button" class="btn btn-primary" type="button" ng-disabled="!selected_node" ng-click="showActions($event)" title="Show actions on selected router">Actions <b class="down caret"></b></button> </div> </div> <div id="topology"><!-- d3 toplogy here --></div> @@ -210,7 +211,7 @@ under the License. </div> <label for="host" class="entity-description">Enter a machine name or IP address</label> <fieldset> - <input type="text" name="host" id="host" ng-model="host" ng-required="true" class="ui-widget-content ui-corner-all"/> + <input type="text" name="host" id="host" ng-model="host" ng-required="true" class="router-host ui-widget-content ui-corner-all"/> </fieldset> </div> @@ -220,3 +221,20 @@ under the License. </div> </form> </script> + +<script type="text/ng-template" id="deploy-template.html"> + <form novalidate> + <div class="modal-header"> + <h3 class="modal-title">{{state}}</h3> + </div> + <div class="modal-body"> + <pre id="deploy_status" class="tail">{{status}}</pre> + <div ng-hide="polling" id="message">Deployed. + <div ng-show="hasConsole()">Browse to <a ng-href="{{address}}" target="_blank">here</a> to manage the router.</div> + </div> + </div> + <div class="modal-footer"> + <button class="btn btn-warning" type="button" ng-click="cancel()">Close</button> + </div> + </form> +</script> http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/js/qdrTopology.js ---------------------------------------------------------------------- diff --git a/console/config/js/qdrTopology.js b/console/config/js/qdrTopology.js index e8b7df8..421191b 100644 --- a/console/config/js/qdrTopology.js +++ b/console/config/js/qdrTopology.js @@ -33,9 +33,26 @@ var QDR = (function(QDR) { var sections = ['log', 'connector', 'sslProfile', 'listener'] $scope.Publish = function () { - doPublish() + doOperation("PUBLISH", function (response) { + Core.notification('info', $scope.mockTopologyDir + " published"); + QDR.log.info("published " + $scope.mockTopologyDir) + }) + } + $scope.Deploy = function () { + // send the deploy command + doOperation("DEPLOY", function (response) { + QDR.log.info("deployment " + $scope.mockTopologyDir + " started") + }) + // show the deploy status dialog + doDeployDialog() + } + + $scope.showConfig = function (node) { + doOperation("SHOW-CONFIG", function (response) { + doShowConfigDialog(response) + }, {nodeIndex: node.index}) } - var doPublish = function (nodeIndex, callback) { + var doOperation = function (operation, callback, extraProps) { var l = [] links.forEach( function (link) { if (link.source.nodeType === 'inter-router' && link.target.nodeType === 'inter-router') @@ -44,27 +61,17 @@ var QDR = (function(QDR) { cls: link.cls}) }) var props = {nodes: nodes, links: l, topology: $scope.mockTopologyDir, settings: settings} - if (angular.isDefined(nodeIndex)) { - op = "SHOW-CONFIG" - props.nodeIndex = nodeIndex - } else { - op = "PUBLISH" - } - QDRService.sendMethod(op, props, function (response) { - if (!angular.isDefined(nodeIndex)) { - Core.notification('info', props.topology + " published"); - QDR.log.info("published " + $scope.mockTopologyDir) - } else { + if (extraProps) + Object.assign(props, props, extraProps) + QDRService.sendMethod(operation, props, function (response) { + if (callback) callback(response) - } - }) - } - $scope.showConfig = function (node) { - doPublish(node.index, function (response) { - doShowConfigDialog(response) }) } + $scope.canDeploy = function () { + return nodes.length > 0 + } $scope.$watch('mockTopologyDir', function(newVal, oldVal) { if (oldVal != newVal) { switchTopology(newVal) @@ -127,7 +134,9 @@ var QDR = (function(QDR) { animate = true QDR.log.info("switched to " + topology) initForceGraph() - Core.notification('info', "switched to " + props.topology); + $timeout( function () { + Core.notification('info', "switched to " + props.topology); + }) }) } @@ -137,7 +146,9 @@ var QDR = (function(QDR) { $scope.selected_node = null resetMouseVars() force.nodes(nodes).links(links).start(); - restart(); + $timeout( function () { + restart(); + }) } $scope.delNode = function (node, skipinit) { @@ -1370,6 +1381,10 @@ var QDR = (function(QDR) { $scope.mockTopologies = [] $scope.mockTopologyDir = "" + $scope.ansible = false + QDRService.sendMethod("ANSIBLE-INSTALLED", {}, function (response) { + $scope.ansible = (response !== "") + }) QDRService.sendMethod("GET-TOPOLOGY-LIST", {}, function (response) { $scope.mockTopologies = response.sort() QDRService.sendMethod("GET-TOPOLOGY", {}, function (response) { @@ -1453,6 +1468,48 @@ var QDR = (function(QDR) { }); }) }; + function doDeployDialog() { + var host = undefined + var port = undefined + for (var i=0; i<nodes.length; i++) { + var node = nodes[i] + if (node.listeners) { + for (var l in node.listeners) { + var listener = node.listeners[l] + if (listener.http) { + host = node.host + port = listener.port + } + } + } + } + var d = $uibModal.open({ + dialogClass: "modal dlg-large", + backdrop: true, + keyboard: true, + backdropClick: true, + controller: 'QDR.DeployDialogController', + templateUrl: 'deploy-template.html', + resolve: { + dir: function () { + return $scope.mockTopologyDir + }, + http_host: function () { + return host + }, + http_port: function () { + return port + } + } + }); + $timeout(function () { + d.result.then(function(result) { + if (result) { + } + }); + }) + } + function doSettingsDialog(opts) { var d = $uibModal.open({ dialogClass: "modal dlg-large", @@ -1674,6 +1731,58 @@ var QDR = (function(QDR) { }) + QDR.module.controller("QDR.DeployDialogController", function($scope, $uibModalInstance, QDRService, $timeout, $sce, dir, http_host, http_port) { + // setup polling to get deployment status + $scope.polling = true + $scope.state = "Deploying" + $scope.address = "" + var pollTimer = null + function doPoll() { + QDRService.sendMethod("DEPLOY-STATUS", {config: dir}, function (response) { + if (response[1] === 'DONE') { + $scope.polling = false + $scope.state = "Deploy Completed" + Core.notification('info', dir + " deployed"); + if (http_host && http_port) { + $scope.address = $sce.trustAsHtml("http://" + http_host + ":" + http_port + "/#!/topology") + } + } + $timeout(function () { + $scope.status = response[0] + scrollToEnd() + if (response[1] === 'DONE') { + } + if ($scope.polling) ( + pollTimer = setTimeout(doPoll, 1000) + ) + }) + }) + } + pollTimer = setTimeout(doPoll, 1000) + $scope.hasConsole = function () { + return http_host && http_port + } + + function scrollTopTween(scrollTop) { + return function() { + var i = d3.interpolateNumber(this.scrollTop, scrollTop); + return function(t) { this.scrollTop = i(t); }; + } + } + var scrollToEnd = function () { + var scrollheight = d3.select("#deploy_status").property("scrollHeight"); + + d3.select('#deploy_status') + .transition().duration(1000) + .tween("uniquetweenname", scrollTopTween(scrollheight)); + } + $scope.cancel = function () { + polling = false + clearTimeout(pollTimer) + $uibModalInstance.close() + } + + }) return QDR; }(QDR || {})); http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/mock/section.py ---------------------------------------------------------------------- diff --git a/console/config/mock/section.py b/console/config/mock/section.py index 79d0601..9df97aa 100644 --- a/console/config/mock/section.py +++ b/console/config/mock/section.py @@ -20,6 +20,7 @@ import json import re from schema import Schema +import pdb class ConfigSection(object): def __init__(self, type, defaults, ignore, opts): @@ -57,6 +58,10 @@ class RouterSection(ConfigSection): super(RouterSection, self).__init__("router", RouterSection.defaults, RouterSection.ignore, kwargs) self.setEntry("id", id) + def __repr__(self): + s = super(RouterSection, self).__repr__() + return s.replace('deploy_host', '#deploy_host', 1) + class ListenerSection(ConfigSection): defaults = {"role": "normal", "host": "0.0.0.0", --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@qpid.apache.org For additional commands, e-mail: commits-h...@qpid.apache.org