GCE Loadbalancer Support and improvements in the compute driver.
Signed-off-by: Tomaz Muraus <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/2115f9f6 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/2115f9f6 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/2115f9f6 Branch: refs/heads/trunk Commit: 2115f9f6934e85c3bc36e4b886c0e3ccc181e34e Parents: 0d05725 Author: Rick Wright <[email protected]> Authored: Thu Oct 3 13:44:29 2013 -0700 Committer: Tomaz Muraus <[email protected]> Committed: Fri Oct 4 16:23:49 2013 +0100 ---------------------------------------------------------------------- demos/gce_demo.py | 16 +- demos/gce_lb_demo.py | 302 ++++ libcloud/common/google.py | 128 +- libcloud/compute/drivers/gce.py | 1310 +++++++++++++++--- libcloud/loadbalancer/drivers/gce.py | 363 +++++ libcloud/loadbalancer/providers.py | 5 +- libcloud/loadbalancer/types.py | 1 + .../gce/aggregated_forwardingRules.json | 57 + .../fixtures/gce/aggregated_targetPools.json | 64 + .../fixtures/gce/global_httpHealthChecks.json | 35 + .../global_httpHealthChecks_basic-check.json | 15 + .../global_httpHealthChecks_lchealthcheck.json | 14 + ...l_httpHealthChecks_lchealthcheck_delete.json | 14 + ...obal_httpHealthChecks_lchealthcheck_put.json | 14 + ...althChecks_libcloud-lb-demo-healthcheck.json | 13 + .../gce/global_httpHealthChecks_post.json | 13 + ...l_httpHealthChecks_lchealthcheck_delete.json | 14 + ...obal_httpHealthChecks_lchealthcheck_put.json | 15 + ..._operation_global_httpHealthChecks_post.json | 14 + ...forwardingRules_lcforwardingrule_delete.json | 16 + ...egions_us-central1_forwardingRules_post.json | 16 + ...tPools_lctargetpool_addHealthCheck_post.json | 16 + ...rgetPools_lctargetpool_addInstance_post.json | 16 + ...entral1_targetPools_lctargetpool_delete.json | 15 + ...ols_lctargetpool_removeHealthCheck_post.json | 16 + ...tPools_lctargetpool_removeInstance_post.json | 16 + ...on_regions_us-central1_targetPools_post.json | 15 + libcloud/test/compute/fixtures/gce/regions.json | 45 + .../regions_us-central1_forwardingRules.json | 31 + ...ntral1_forwardingRules_lcforwardingrule.json | 12 + ...forwardingRules_lcforwardingrule_delete.json | 15 + ...al1_forwardingRules_libcloud-lb-demo-lb.json | 12 + ...egions_us-central1_forwardingRules_post.json | 14 + .../gce/regions_us-central1_targetPools.json | 38 + ...ns_us-central1_targetPools_lctargetpool.json | 15 + ...tPools_lctargetpool_addHealthCheck_post.json | 15 + ...rgetPools_lctargetpool_addInstance_post.json | 15 + ...entral1_targetPools_lctargetpool_delete.json | 15 + ...ols_lctargetpool_removeHealthCheck_post.json | 15 + ...tPools_lctargetpool_removeInstance_post.json | 15 + ...ral1_targetPools_libcloud-lb-demo-lb-tp.json | 16 + .../regions_us-central1_targetPools_post.json | 14 + ...egions_us-central1_targetPools_www-pool.json | 17 + ...l1-b_instances_libcloud-lb-demo-www-000.json | 52 + ...l1-b_instances_libcloud-lb-demo-www-001.json | 52 + ...l1-b_instances_libcloud-lb-demo-www-002.json | 13 + libcloud/test/compute/test_gce.py | 423 +++++- libcloud/test/loadbalancer/test_gce.py | 207 +++ 48 files changed, 3341 insertions(+), 243 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/demos/gce_demo.py ---------------------------------------------------------------------- diff --git a/demos/gce_demo.py b/demos/gce_demo.py index c2209de..631db56 100755 --- a/demos/gce_demo.py +++ b/demos/gce_demo.py @@ -41,7 +41,13 @@ import sys try: import secrets except ImportError: - secrets = None + print('"demos/secrets.py" not found.\n\n' + 'Please copy secrets.py-dist to secrets.py and update the GCE* ' + 'values with appropriate authentication information.\n' + 'Additional information about setting these values can be found ' + 'in the docstring for:\n' + 'libcloud/common/google.py\n') + sys.exit(1) # Add parent dir of this file's dir to sys.path (OS-agnostically) sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__), @@ -58,7 +64,7 @@ MAX_NODES = 5 DEMO_BASE_NAME = 'libcloud-demo' # Datacenter to create resources in -DATACENTER = 'us-central1-a' +DATACENTER = 'us-central2-a' # Clean up resources at the end (can be set to false in order to # inspect resources at the end of the run). Resources will be cleaned @@ -68,10 +74,14 @@ CLEANUP = True args = getattr(secrets, 'GCE_PARAMS', ()) kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {}) +# Add datacenter to kwargs for Python 2.5 compatibility +kwargs = kwargs.copy() +kwargs['datacenter'] = DATACENTER + # ==== HELPER FUNCTIONS ==== def get_gce_driver(): - driver = get_driver(Provider.GCE)(*args, datacenter=DATACENTER, **kwargs) + driver = get_driver(Provider.GCE)(*args, **kwargs) return driver http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/demos/gce_lb_demo.py ---------------------------------------------------------------------- diff --git a/demos/gce_lb_demo.py b/demos/gce_lb_demo.py new file mode 100755 index 0000000..6744095 --- /dev/null +++ b/demos/gce_lb_demo.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This example performs several tasks on Google Compute Engine and the GCE +# Load Balancer. It can be run directly or can be imported into an +# interactive python session. This can also serve as an integration test for +# the GCE Load Balancer Driver. +# +# To run interactively: +# - Make sure you have valid values in secrets.py +# (For more information about setting up your credentials, see the +# libcloud/common/google.py docstring) +# - Run 'python' in this directory, then: +# import gce_lb_demo +# gcelb = gce_lb_demo.get_gcelb_driver() +# gcelb.list_balancers() +# etc. +# - Or, to run the full demo from the interactive python shell: +# import gce_lb_demo +# gce_lb_demo.CLEANUP = False # optional +# gce_lb_demo.MAX_NODES = 4 # optional +# gce_lb_demo.DATACENTER = 'us-central1-a' # optional +# gce_lb_demo.main() + +import os.path +import sys +import time + +try: + import secrets +except ImportError: + print('"demos/secrets.py" not found.\n\n' + 'Please copy secrets.py-dist to secrets.py and update the GCE* ' + 'values with appropriate authentication information.\n' + 'Additional information about setting these values can be found ' + 'in the docstring for:\n' + 'libcloud/common/google.py\n') + sys.exit(1) + +# Add parent dir of this file's dir to sys.path (OS-agnostically) +sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__), + os.path.pardir))) + +from libcloud.utils.py3 import PY3 +if PY3: + import urllib.request as url_req +else: + import urllib2 as url_req + +# This demo uses both the Compute driver and the LoadBalancer driver +from libcloud.compute.types import Provider +from libcloud.compute.providers import get_driver +from libcloud.loadbalancer.types import Provider as Provider_lb +from libcloud.loadbalancer.providers import get_driver as get_driver_lb + +# String that all resource names created by the demo will start with +# WARNING: Any resource that has a matching name will be destroyed. +DEMO_BASE_NAME = 'libcloud-lb-demo' + +# Datacenter to create resources in +DATACENTER = 'us-central1-b' + +# Clean up resources at the end (can be set to false in order to +# inspect resources at the end of the run). Resources will be cleaned +# at the beginning regardless. +CLEANUP = True + +args = getattr(secrets, 'GCE_PARAMS', ()) +kwargs = getattr(secrets, 'GCE_KEYWORD_PARAMS', {}) + +# Add datacenter to kwargs for Python 2.5 compatibility +kwargs = kwargs.copy() +kwargs['datacenter'] = DATACENTER + + +# ==== HELPER FUNCTIONS ==== +def get_gce_driver(): + driver = get_driver(Provider.GCE)(*args, **kwargs) + return driver + + +def get_gcelb_driver(gce_driver=None): + # The GCE Load Balancer driver uses the GCE Compute driver for all of its + # API calls. You can either provide the driver directly, or provide the + # same authentication information so the the LB driver can get its own + # Compute driver. + if gce_driver: + driver = get_driver_lb(Provider_lb.GCE)(gce_driver=gce_driver) + else: + driver = get_driver_lb(Provider_lb.GCE)(*args, **kwargs) + return driver + + +def display(title, resource_list): + """ + Display a list of resources. + + :param title: String to be printed at the heading of the list. + :type title: ``str`` + + :param resource_list: List of resources to display + :type resource_list: Any ``object`` with a C{name} attribute + """ + print('%s:' % title) + for item in resource_list[:10]: + print(' %s' % item.name) + + +def clean_up(base_name, node_list=None, resource_list=None): + """ + Destroy all resources that have a name beginning with 'base_name'. + + :param base_name: String with the first part of the name of resources + to destroy + :type base_name: ``str`` + + :keyword node_list: List of nodes to consider for deletion + :type node_list: ``list`` of :class:`Node` + + :keyword resource_list: List of resources to consider for deletion + :type resource_list: ``list`` of I{Resource Objects} + """ + if node_list is None: + node_list = [] + if resource_list is None: + resource_list = [] + # Use ex_destroy_multiple_nodes to destroy nodes + del_nodes = [] + for node in node_list: + if node.name.startswith(base_name): + del_nodes.append(node) + + result = gce.ex_destroy_multiple_nodes(del_nodes) + for i, success in enumerate(result): + if success: + print(' Deleted %s' % del_nodes[i].name) + else: + print(' Failed to delete %s' % del_nodes[i].name) + + # Destroy everything else with just the destroy method + for resource in resource_list: + if resource.name.startswith(base_name): + if resource.destroy(): + print(' Deleted %s' % resource.name) + else: + print(' Failed to Delete %s' % resource.name) + + +# ==== DEMO CODE STARTS HERE ==== +def main(): + global gce # Used by the clean_up function + gce = get_gce_driver() + gcelb = get_gcelb_driver(gce) + + # Existing Balancers + balancers = gcelb.list_balancers() + display('Load Balancers', balancers) + + # Protocols + protocols = gcelb.list_protocols() + print('Protocols:') + for p in protocols: + print(' %s' % p) + + # Healthchecks + healthchecks = gcelb.ex_list_healthchecks() + display('Health Checks', healthchecks) + + # This demo is based on the GCE Load Balancing Quickstart described here: + # https://developers.google.com/compute/docs/load-balancing/lb-quickstart + + # == Clean-up and existing demo resources == + all_nodes = gce.list_nodes(ex_zone='all') + firewalls = gce.ex_list_firewalls() + print('Cleaning up any "%s" resources:' % DEMO_BASE_NAME) + clean_up(DEMO_BASE_NAME, all_nodes, balancers + healthchecks + firewalls) + + # == Create 3 nodes to balance between == + startup_script = ('apt-get -y update && ' + 'apt-get -y install apache2 && ' + 'hostname > /var/www/index.html') + tag = '%s-www' % DEMO_BASE_NAME + base_name = '%s-www' % DEMO_BASE_NAME + image = gce.ex_get_image('debian-7') + size = gce.ex_get_size('n1-standard-1') + number = 3 + metadata = {'items': [{'key': 'startup-script', + 'value': startup_script}]} + lb_nodes = gce.ex_create_multiple_nodes(base_name, size, image, + number, ex_tags=[tag], + ex_metadata=metadata) + display('Created Nodes', lb_nodes) + + # == Create a Firewall for instances == + print('Creating a Firewall:') + name = '%s-firewall' % DEMO_BASE_NAME + allowed = [{'IPProtocol': 'tcp', + 'ports': ['80']}] + firewall = gce.ex_create_firewall(name, allowed, source_tags=[tag]) + print(' Firewall %s created' % firewall.name) + + # == Create a Health Check == + print('Creating a HealthCheck:') + name = '%s-healthcheck' % DEMO_BASE_NAME + + # These are all the default values, but listed here as an example. To + # create a healthcheck with the defaults, only name is required. + hc = gcelb.ex_create_healthcheck(name, host=None, path='/', port='80', + interval=5, timeout=5, + unhealthy_threshold=2, + healthy_threshold=2) + print(' Healthcheck %s created' % hc.name) + + # == Create Load Balancer == + print('Creating Load Balancer') + name = '%s-lb' % DEMO_BASE_NAME + port = 80 + protocol = 'tcp' + algorithm = None + members = lb_nodes[:2] # Only attach the first two initially + healthchecks = [hc] + balancer = gcelb.create_balancer(name, port, protocol, algorithm, members, + ex_healthchecks=healthchecks) + print(' Load Balancer %s created' % balancer.name) + + # == Attach third Node == + print('Attaching additional node to Load Balancer:') + member = balancer.attach_compute_node(lb_nodes[2]) + print(' Attached %s to %s' % (member.id, balancer.name)) + + # == Show Balancer Members == + members = balancer.list_members() + print('Load Balancer Members:') + for member in members: + print(' ID: %s IP: %s' % (member.id, member.ip)) + + # == Remove a Member == + print('Removing a Member:') + detached = members[0] + detach = balancer.detach_member(detached) + if detach: + print(' Member %s detached from %s' % (detached.id, balancer.name)) + + # == Show Updated Balancer Members == + members = balancer.list_members() + print('Updated Load Balancer Members:') + for member in members: + print(' ID: %s IP: %s' % (member.id, member.ip)) + + # == Reattach Member == + print('Reattaching Member:') + member = balancer.attach_member(detached) + print(' Member %s attached to %s' % (member.id, balancer.name)) + + # == Test Load Balancer by connecting to it multiple times == + print('Sleeping for 10 seconds to stabilize the balancer...') + time.sleep(10) + rounds = 200 + url = 'http://%s/' % balancer.ip + line_length = 75 + print('Connecting to %s %s times:' % (url, rounds)) + for x in range(rounds): + response = url_req.urlopen(url) + if PY3: + output = str(response.read(), encoding='utf-8').strip() + else: + output = response.read().strip() + if 'www-001' in output: + padded_output = output.center(line_length) + elif 'www-002' in output: + padded_output = output.rjust(line_length) + else: + padded_output = output.ljust(line_length) + sys.stdout.write('\r%s' % padded_output) + sys.stdout.flush() + print('') + + if CLEANUP: + balancers = gcelb.list_balancers() + healthchecks = gcelb.ex_list_healthchecks() + nodes = gce.list_nodes(ex_zone='all') + firewalls = gce.ex_list_firewalls() + + print('Cleaning up %s resources created.' % DEMO_BASE_NAME) + clean_up(DEMO_BASE_NAME, nodes, balancers + healthchecks + firewalls) + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/libcloud/blob/2115f9f6/libcloud/common/google.py ---------------------------------------------------------------------- diff --git a/libcloud/common/google.py b/libcloud/common/google.py index 2a60251..fb668c3 100644 --- a/libcloud/common/google.py +++ b/libcloud/common/google.py @@ -71,13 +71,15 @@ import time import datetime import os import socket +import sys -from libcloud.utils.py3 import urlencode, urlparse, PY3 +from libcloud.utils.py3 import httplib, urlencode, urlparse, PY3 from libcloud.common.base import (ConnectionUserAndKey, JsonResponse, PollingConnection) -from libcloud.compute.types import (InvalidCredsError, - MalformedResponseError, - LibcloudError) +from libcloud.common.types import (InvalidCredsError, + MalformedResponseError, + ProviderError, + LibcloudError) try: from Crypto.Hash import SHA256 @@ -101,10 +103,121 @@ class GoogleAuthError(LibcloudError): return repr(self.value) -class GoogleResponse(JsonResponse): +class GoogleBaseError(ProviderError): + def __init__(self, value, http_code, code, driver=None): + self.code = code + super(GoogleBaseError, self).__init__(value, http_code, driver) + + +class JsonParseError(GoogleBaseError): + pass + + +class ResourceNotFoundError(GoogleBaseError): + pass + + +class QuotaExceededError(GoogleBaseError): + pass + + +class ResourceExistsError(GoogleBaseError): pass +class ResourceInUseError(GoogleBaseError): + pass + + +class GoogleResponse(JsonResponse): + """ + Google Base Response class. + """ + def success(self): + """ + Determine if the request was successful. + + For the Google response class, tag all responses as successful and + raise appropriate Exceptions from parse_body. + + :return: C{True} + """ + return True + + def _get_error(self, body): + """ + Get the error code and message from a JSON response. + + Return just the first error if there are multiple errors. + + :param body: The body of the JSON response dictionary + :type body: ``dict`` + + :return: Tuple containing error code and message + :rtype: ``tuple`` of ``str`` or ``int`` + """ + if 'errors' in body['error']: + err = body['error']['errors'][0] + else: + err = body['error'] + + code = err.get('code') + message = err.get('message') + return (code, message) + + def parse_body(self): + """ + Parse the JSON response body, or raise exceptions as appropriate. + + :return: JSON dictionary + :rtype: ``dict`` + """ + if len(self.body) == 0 and not self.parse_zero_length_body: + return self.body + + json_error = False + try: + body = json.loads(self.body) + except: + # If there is both a JSON parsing error and an unsuccessful http + # response (like a 404), we want to raise the http error and not + # the JSON one, so don't raise JsonParseError here. + body = self.body + json_error = True + + if self.status in [httplib.OK, httplib.CREATED, httplib.ACCEPTED]: + if json_error: + raise JsonParseError(body, self.status, None) + elif 'error' in body: + (code, message) = self._get_error(body) + if code == 'QUOTA_EXCEEDED': + raise QuotaExceededError(message, self.status, code) + elif code == 'RESOURCE_ALREADY_EXISTS': + raise ResourceExistsError(message, self.status, code) + elif code.startswith('RESOURCE_IN_USE'): + raise ResourceInUseError(message, self.status, code) + else: + raise GoogleBaseError(message, self.status, code) + else: + return body + + elif self.status == httplib.NOT_FOUND: + if (not json_error) and ('error' in body): + (code, message) = self._get_error(body) + else: + message = body + code = None + raise ResourceNotFoundError(message, self.status, code) + + else: + if (not json_error) and ('error' in body): + (code, message) = self._get_error(body) + else: + message = body + code = None + raise GoogleBaseError(message, self.status, code) + + class GoogleBaseDriver(object): name = "Google API" @@ -395,6 +508,11 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): super(GoogleBaseConnection, self).__init__(user_id, key, **kwargs) + python_ver = '%s.%s.%s' % (sys.version_info[0], sys.version_info[1], + sys.version_info[2]) + ver_platform = 'Python %s/%s' % (python_ver, sys.platform) + self.user_agent_append(ver_platform) + def _now(self): return datetime.datetime.utcnow()
