Initial checkin for Python Utilities and SDK
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/32f9e55d Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/32f9e55d Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/32f9e55d Branch: refs/heads/master Commit: 32f9e55da01c92dbda81a88a465f088405259faa Parents: 91abfd8 Author: Jeff West <jw...@apigee.com> Authored: Tue Jul 26 13:26:36 2016 -0700 Committer: Jeff West <jw...@apigee.com> Committed: Tue Jul 26 13:26:36 2016 -0700 ---------------------------------------------------------------------- sdks/python/.gitignore | 57 + sdks/python/LICENSE | 202 ++ sdks/python/README.md | 16 + sdks/python/sample_app.py | 68 + sdks/python/setup.py | 39 + sdks/python/usergrid/UsergridApplication.py | 62 + sdks/python/usergrid/UsergridAuth.py | 103 + sdks/python/usergrid/UsergridClient.py | 398 ++++ sdks/python/usergrid/UsergridCollection.py | 77 + sdks/python/usergrid/UsergridConnection.py | 26 + sdks/python/usergrid/UsergridError.py | 17 + sdks/python/usergrid/UsergridOrganization.py | 31 + sdks/python/usergrid/UsergridQueryIterator.py | 155 ++ sdks/python/usergrid/__init__.py | 37 + sdks/python/usergrid/app_templates.py | 36 + sdks/python/usergrid/management_templates.py | 25 + utils/usergrid-util-python/.gitignore | 61 + utils/usergrid-util-python/LICENSE | 22 + utils/usergrid-util-python/README.md | 15 + .../es_tools/alias_mover.py | 129 ++ .../es_tools/cluster_shard_allocation.py | 89 + .../es_tools/command_sender.py | 42 + .../es_tools/es_index_iterator_reindexer.py | 107 + .../es_tools/es_searcher.py | 24 + .../es_tools/index_deleter.py | 68 + .../es_tools/index_iterator_size_checker.py | 270 +++ .../es_tools/index_prefix_checker.py | 81 + .../es_tools/index_replica_setter.py | 118 + .../es_tools/index_shard_allocator.py | 148 ++ .../es_tools/mapping_deleter.py | 34 + .../es_tools/mapping_retriever.py | 45 + .../es_tools/monitor_tasks.py | 41 + utils/usergrid-util-python/index_test/README.md | 1 + .../index_test/document_creator.py | 254 ++ .../index_test/index_test_mixed_batch.py | 545 +++++ .../index_test/index_test_single_type_batch.py | 547 +++++ utils/usergrid-util-python/requirements.txt | 4 + .../activity_streams/activity_streams.py | 132 ++ .../samples/beacon-event-example.py | 196 ++ .../samples/counter_test.py | 31 + utils/usergrid-util-python/setup.py | 40 + .../usergrid_tools/__init__.py | 4 + .../usergrid_tools/general/__init__.py | 0 .../usergrid_tools/general/deleter.py | 151 ++ .../general/duplicate_name_checker.py | 25 + .../usergrid_tools/general/queue_monitor.py | 119 + .../usergrid_tools/general/url_tester.py | 87 + .../general/user_confirm_activate.py | 29 + .../usergrid_tools/general/user_creator.py | 49 + .../usergrid_tools/groups/__init__.py | 0 .../usergrid_tools/groups/big_group_creater.py | 86 + .../usergrid_tools/indexing/README.md | 22 + .../usergrid_tools/indexing/__init__.py | 0 .../usergrid_tools/indexing/batch_index_test.py | 340 +++ .../indexing/entity_index_test.py | 317 +++ .../usergrid_tools/iterators/README.md | 8 + .../usergrid_tools/iterators/__init__.py | 0 .../usergrid_tools/iterators/simple_iterator.py | 79 + .../iterators/usergrid_cross_region_iterator.py | 409 ++++ .../usergrid_tools/library_check.py | 23 + .../usergrid_tools/migration/README.md | 234 ++ .../usergrid_tools/migration/__init__.py | 2 + .../migration/usergrid_data_exporter.py | 923 ++++++++ .../migration/usergrid_data_migrator.py | 2168 ++++++++++++++++++ .../usergrid_tools/parse_importer/README.md | 90 + .../usergrid_tools/parse_importer/__init__.py | 0 .../parse_importer/parse_importer.py | 385 ++++ .../usergrid_tools/permissions/README.md | 3 + .../usergrid_tools/permissions/permissions.py | 146 ++ .../usergrid_tools/queue/README.md | 1 + .../queue/dlq-iterator-checker.py | 143 ++ .../usergrid_tools/queue/dlq_requeue.py | 173 ++ .../queue/queue-config-sample.json | 22 + .../usergrid_tools/queue/queue_cleaner.py | 155 ++ .../usergrid_tools/redis/redis_iterator.py | 30 + .../usergrid_tools/redis/redisscan.py | 15 + 76 files changed, 10631 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/.gitignore ---------------------------------------------------------------------- diff --git a/sdks/python/.gitignore b/sdks/python/.gitignore new file mode 100644 index 0000000..ba74660 --- /dev/null +++ b/sdks/python/.gitignore @@ -0,0 +1,57 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/LICENSE ---------------------------------------------------------------------- diff --git a/sdks/python/LICENSE b/sdks/python/LICENSE new file mode 100755 index 0000000..8f71f43 --- /dev/null +++ b/sdks/python/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/README.md ---------------------------------------------------------------------- diff --git a/sdks/python/README.md b/sdks/python/README.md new file mode 100755 index 0000000..cc2af97 --- /dev/null +++ b/sdks/python/README.md @@ -0,0 +1,16 @@ +# Usergrid Python SDK + +# Overview +This is a starter project for the Usergrid Python SDK. It is a work in progress. + +# Installation + +## PIP (http://pip.readthedocs.org/en/stable/installing/) + +`pip install usergrid` + +## Manual installation + +- `git clone g...@github.com:jwest-apigee/usergrid-python.git` +- `cd usergrid-python` +- `pip install -e .` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/sample_app.py ---------------------------------------------------------------------- diff --git a/sdks/python/sample_app.py b/sdks/python/sample_app.py new file mode 100755 index 0000000..a829736 --- /dev/null +++ b/sdks/python/sample_app.py @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +from usergrid import Usergrid + +__author__ = 'ApigeeCorporation' + + +def main(): + Usergrid.init(org_id='jwest1', + app_id='sandbox') + + response = Usergrid.DELETE('pets', 'max') + if not response.ok: + print 'Failed to delete max: %s' % response + exit() + + response = Usergrid.DELETE('owners', 'jeff') + if not response.ok: + print 'Failed to delete Jeff: %s' % response + exit() + + response = Usergrid.POST('pets', {'name': 'max'}) + + if response.ok: + pet = response.first() + print pet + response = Usergrid.POST('owners', {'name': 'jeff'}) + + if response.ok: + owner = response.first() + print owner + response = pet.connect('ownedBy', owner) + + if response.ok: + print 'Connected!' + + response = pet.disconnect('ownedBy', owner) + + if response.ok: + print 'all done!' + else: + print response + else: + print 'failed to connect: %s' % response + + else: + print 'Failed to create Jeff: %s' % response + + else: + print response + + +main() http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/setup.py ---------------------------------------------------------------------- diff --git a/sdks/python/setup.py b/sdks/python/setup.py new file mode 100755 index 0000000..6eec51f --- /dev/null +++ b/sdks/python/setup.py @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +__author__ = 'Jeff West @ ApigeeCorporation' + +from setuptools import setup, find_packages + +VERSION = '0.1.11' + +setup( + name='usergrid', + version=VERSION, + description='Usergrid SDK for Python', + url='http://usergrid.apache.org', + download_url="https://codeload.github.com/jwest-apigee/usergrid-python/zip/v"; + VERSION, + author='Jeff West', + author_email='jw...@apigee.com', + packages=find_packages(), + install_requires=[ + 'requests', + 'urllib3' + ], + entry_points={ + } +) http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridApplication.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridApplication.py b/sdks/python/usergrid/UsergridApplication.py new file mode 100644 index 0000000..cedd5b1 --- /dev/null +++ b/sdks/python/usergrid/UsergridApplication.py @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +import logging +from usergrid import UsergridError, UsergridCollection +from usergrid.app_templates import app_url_template + + +class UsergridApplication(object): + def __init__(self, app_id, client): + self.app_id = app_id + self.client = client + self.logger = logging.getLogger('usergrid.UsergridClient') + + def list_collections(self): + url = app_url_template.format(app_id=self.app_id, + **self.client.url_data) + r = self.client.get(url) + + if r.status_code == 200: + api_response = r.json() + collection_list = api_response.get('entities')[0].get('metadata', {}).get('collections', {}) + collections = {} + + for collection_name in collection_list: + collections[collection_name] = UsergridCollection(self.client.org_id, + self.app_id, + collection_name, + self.client) + + return collections + + else: + raise UsergridError(message='Unable to post to list collections', + status_code=r.status_code, + api_response=r, + url=url) + + def collection(self, collection_name): + return UsergridCollection(self.client.org_id, + self.app_id, + collection_name, + self.client) + + def authenticate_app_client(self, + **kwargs): + + return self.client.authenticate_app_client(self.app_id, **kwargs) http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridAuth.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridAuth.py b/sdks/python/usergrid/UsergridAuth.py new file mode 100644 index 0000000..3406312 --- /dev/null +++ b/sdks/python/usergrid/UsergridAuth.py @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +import json +import requests +from usergrid.management_templates import org_token_url_template + + +class UsergridAuth: + def __init__(self, + grant_type, + url_template, + username=None, + password=None, + client_id=None, + client_secret=None, + token_ttl_seconds=86400): + + self.grant_type = grant_type + self.username = username + self.password = password + self.client_id = client_id + self.client_secret = client_secret + self.token_ttl_seconds = token_ttl_seconds + self.url_template = url_template + self.access_token = None + + def get_token_request(self): + if self.grant_type == 'client_credentials': + return { + 'grant_type': 'client_credentials', + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'ttl': self.token_ttl_seconds * 1000 + } + elif self.grant_type == 'password': + return { + 'grant_type': 'password', + 'username': self.username, + 'password': self.password, + 'ttl': self.token_ttl_seconds * 1000 + } + + else: + raise ValueError('Unspecified/unknown grant type: %s' % self.grant_type) + + def authenticate(self, client): + token_request = self.get_token_request() + + url = self.url_template.format(**client.url_data) + + r = requests.post(url, data=json.dumps(token_request)) + + if r.status_code == 200: + response = r.json() + self.access_token = response.get('access_token') + + else: + raise ValueError('Unable to authenticate: %s' % r.text) + + +class UsergridOrgAuth(UsergridAuth): + def __init__(self, client_id, client_secret, token_ttl_seconds=86400): + UsergridAuth.__init__(self, + grant_type='client_credentials', + url_template=org_token_url_template, + client_id=client_id, + client_secret=client_secret, + token_ttl_seconds=token_ttl_seconds) + + +class UsergridAppAuth(UsergridAuth): + def __init__(self, client_id, client_secret, token_ttl_seconds=86400): + UsergridAuth.__init__(self, + grant_type='client_credentials', + url_template=app_token_url_template, + client_id=client_id, + client_secret=client_secret, + token_ttl_seconds=token_ttl_seconds) + + +class UsergridUserAuth(UsergridAuth): + def __init__(self, username, password, token_ttl_seconds=86400): + UsergridAuth.__init__(self, + grant_type='password', + url_template=app_token_url_template, + username=username, + password=password, + token_ttl_seconds=token_ttl_seconds) http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridClient.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridClient.py b/sdks/python/usergrid/UsergridClient.py new file mode 100644 index 0000000..2ab8f73 --- /dev/null +++ b/sdks/python/usergrid/UsergridClient.py @@ -0,0 +1,398 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +import json +import logging +import requests +from usergrid.UsergridAuth import UsergridAppAuth +from usergrid.app_templates import get_entity_url_template, post_collection_url_template, put_entity_url_template, \ + delete_entity_url_template, connect_entities_by_type_template, assign_role_url_template + + +def value_error(message): + raise ValueError(message) + + +def usergrid_error(r): + pass + + +class Usergrid(object): + client = None + + @staticmethod + def init(org_id, + app_id, + **kwargs): + Usergrid.client = UsergridClient(org_id, app_id, **kwargs) + + @staticmethod + def GET(collection, uuid_name, **kwargs): + return Usergrid.client.GET(collection, uuid_name, **kwargs) + + @staticmethod + def PUT(collection, uuid_name, data, **kwargs): + return Usergrid.client.PUT(collection, uuid_name, data, **kwargs) + + @staticmethod + def POST(collection, data, **kwargs): + return Usergrid.client.POST(collection, data, **kwargs) + + @staticmethod + def DELETE(collection, uuid_name, **kwargs): + return Usergrid.client.DELETE(collection, uuid_name, **kwargs) + + @staticmethod + def connect_entities(from_entity, relationship, to_entity, **kwargs): + return Usergrid.client.connect_entities(from_entity, relationship, to_entity, **kwargs) + + @staticmethod + def disconnect_entities(from_entity, relationship, to_entity, **kwargs): + return Usergrid.client.disconnect_entities(from_entity, relationship, to_entity, **kwargs) + + @staticmethod + def assign_role(role_uuid_name, user_entity, **kwargs): + return Usergrid.client.assign_role(role_uuid_name, user_entity, **kwargs) + + +class UsergridResponse(object): + def __init__(self, api_response, client): + self.api_response = api_response + self.client = client + + if api_response is None: + self.ok = False + self.body = 'No Response' + + else: + self.headers = api_response.headers + + if api_response.status_code == 200: + self.ok = True + self.body = api_response.json() + self.entities = self.body.get('entities', []) + + else: + self.ok = False + + if api_response.headers.get('Content-type') == 'application/json': + self.body = api_response.json() + else: + self.body = 'HTTP %s: %s' % (api_response.status_code, api_response.text) + + def __str__(self): + return json.dumps(self.body) + + def first(self): + return UsergridEntity(entity_data=self.entities[0]) if self.ok and self.entities and len( + self.entities) > 0 else None + + def entity(self): + return self.first() + + def last(self): + return UsergridEntity(entity_data=self.entities[len(self.entities) - 1]) if self.ok and self.entities and len( + self.entities) > 0 else None + + def has_next_page(self): + return 'cursor' in self.body if self.ok else False + + +class UsergridEntity(object): + def __init__(self, entity_data): + self.entity_data = entity_data + + def __str__(self): + return json.dumps(self.entity_data) + + def get(self, name, default=None): + return self.entity_data.get(name, default) + + def entity_id(self): + + if self.entity_data.get('type', '').lower() in ['users', 'user']: + return self.entity_data.get('uuid', self.entity_data.get('username')) + + return self.entity_data.get('uuid', self.entity_data.get('name')) + + def can_mutate_or_load(self): + entity_id = self.entity_id() + + if entity_id is None or self.entity_data.get('type') is None: + return False + + return True + + def put_property(self, name, value): + self.entity_data[name] = value + + def put_properties(self, properties): + if isinstance(properties, dict): + self.entity_data.update(properties) + + def remove_property(self, name): + + if name is not None and name in self.entity_data: + del self.entity_data[name] + + def remove_properties(self, properties): + if isinstance(properties, (list, dict)): + for property_name in properties: + self.remove_property(property_name) + + def append(self, array_name, value): + if array_name in self.entity_data: + if isinstance(self.entity_data[array_name], list): + self.entity_data[array_name].append(value) + else: + self.entity_data[array_name] = [value] + + def prepend(self, array_name, value): + if array_name in self.entity_data: + if isinstance(self.entity_data[array_name], list): + self.entity_data[array_name].pre(value) + else: + self.entity_data[array_name] = [value] + + def insert(self, array_name, value, index): + if array_name in self.entity_data: + if isinstance(self.entity_data[array_name], list): + self.entity_data[array_name].insert(index, value) + + def shift(self, array_name): + if array_name in self.entity_data: + if isinstance(self.entity_data[array_name], list): + value = self.entity_data[array_name][0] + self.entity_data[array_name] = self.entity_data[array_name][1:] + return value + + return None + + def reload(self): + if not self.can_mutate_or_load(): + raise ValueError('Unable to reload entity: No uuid nor name') + + response = Usergrid.GET(collection=self.entity_data.get('type'), + uuid_name=self.entity_id()) + if response.ok: + self.entity_data.update(response.entity().entity_data) + + else: + raise ValueError('Unable to reload entity: %s' % response) + + def save(self): + if not self.can_mutate_or_load(): + raise ValueError('Unable to save entity: No uuid nor name') + + response = Usergrid.PUT(collection=self.entity_data.get('type'), + uuid_name=self.entity_id(), + data=self.entity_data) + + if response.ok and 'uuid' not in self.entity_data: + self.entity_data['uuid'] = response.entity().get('uuid') + + return response + + def remove(self): + if not self.can_mutate_or_load(): + raise ValueError('Unable to delete entity: No uuid nor name') + + return Usergrid.DELETE(collection=self.entity_data.get('type'), + uuid_name=self.entity_id()) + + def get_connections(self, relationship, direction='connecting'): + pass + + def connect(self, relationship, to_entity): + + if not to_entity.can_mutate_or_load(): + raise ValueError('Unable to connect to entity - no uuid or name') + + if not self.can_mutate_or_load(): + raise ValueError('Unable from connect to entity - no uuid or name') + + return Usergrid.connect_entities(self, relationship, to_entity) + + def disconnect(self, relationship, to_entity): + if not to_entity.can_mutate_or_load(): + raise ValueError('Unable to connect to entity - no uuid or name') + + if not self.can_mutate_or_load(): + raise ValueError('Unable from connect to entity - no uuid or name') + + return Usergrid.disconnect_entities(self, relationship, to_entity) + + def attach_asset(self, filename, data, content_type): + pass + + def download_asset(self, content_type=None): + pass + + +class UsergridClient(object): + def __init__(self, + org_id, + app_id, + base_url='http://api.usergrid.com', + client_id=None, + client_secret=None, + token_ttl_seconds=86400, + auth_fallback="none"): + + self.base_url = base_url + self.org_id = org_id + self.app_id = app_id + self.auth_fallback = auth_fallback + self.logger = logging.getLogger('usergrid.UsergridClient') + self.session = requests.Session() + + self.url_data = { + 'base_url': base_url, + 'org_id': org_id, + 'app_id': app_id + } + + if client_id and not client_secret: + value_error('Client ID Specified but not Secret') + + elif client_secret and not client_id: + value_error('Client ID Specified but not Secret') + + elif client_secret and client_id: + self.auth = UsergridAppAuth(client_id=client_id, + client_secret=client_secret, + token_ttl_seconds=token_ttl_seconds) + + self.auth.authenticate(self) + self.session.headers.update({'Authorization': 'Bearer %s' % self.auth.access_token}) + + def __str__(self): + return json.dumps({ + 'base_url': self.base_url, + 'org_id': self.org_id, + 'app_id': self.app_id, + 'access_token': self.auth.access_token + }) + + def GET(self, collection, uuid_name, connections='none', auth=None, **kwargs): + url = get_entity_url_template.format(collection=collection, + uuid_name=uuid_name, + connections=connections, + **self.url_data) + if auth: + r = requests.get(url, headers={'Authorization': 'Bearer %s' % auth.access_token}) + + else: + r = self.session.get(url) + + return UsergridResponse(r, self) + + def PUT(self, collection, uuid_name, data, auth=None, **kwargs): + url = put_entity_url_template.format(collection=collection, + uuid_name=uuid_name, + **self.url_data) + + if auth: + r = requests.put(url, + data=json.dumps(data), + headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.put(url, data=json.dumps(data)) + + return UsergridResponse(r, self) + + def POST(self, collection, data, auth=None, **kwargs): + url = post_collection_url_template.format(collection=collection, + **self.url_data) + + if auth: + r = requests.post(url, + data=json.dumps(data), + headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.post(url, data=json.dumps(data)) + + return UsergridResponse(r, self) + + def DELETE(self, collection, uuid_name, auth=None, **kwargs): + url = delete_entity_url_template.format(collection=collection, + uuid_name=uuid_name, + **self.url_data) + + if auth: + r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.delete(url) + + return UsergridResponse(r, self) + + def connect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs): + + url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'), + from_uuid_name=from_entity.entity_id(), + relationship=relationship, + to_collection=to_entity.get('type'), + to_uuid_name=to_entity.entity_id(), + **self.url_data) + + if auth: + r = requests.post(url, headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.post(url) + + return UsergridResponse(r, self) + + def assign_role(self, role_uuid_name, entity, auth=None, **kwargs): + url = assign_role_url_template.format(role_uuid_name=role_uuid_name, + entity_type=entity.get('type'), + entity_uuid_name=entity.entity_id(), + **self.url_data) + + if auth: + r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.delete(url) + + return UsergridResponse(r, self) + + +def disconnect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs): + url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'), + from_uuid_name=from_entity.entity_id(), + relationship=relationship, + to_collection=to_entity.get('type'), + to_uuid_name=to_entity.entity_id(), + **self.url_data) + + if auth: + r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token}) + else: + r = self.session.delete(url) + + return UsergridResponse(r, self) + + +class UsergridUser(object): + def __init__(self): + pass + + +class UsergridAsset(object): + def __init__(self, filename, data, content_type): + self.filename = filename + self.data = data + self.content_type = content_type http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridCollection.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridCollection.py b/sdks/python/usergrid/UsergridCollection.py new file mode 100644 index 0000000..a37ad95 --- /dev/null +++ b/sdks/python/usergrid/UsergridCollection.py @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +class UsergridCollection(object): + def __init__(self, org_id, app_id, collection_name, client): + self.org_id = org_id + self.app_id = app_id + self.collection_name = collection_name + self.client = client + + def __str__(self): + return json.dumps({ + 'org_id': self.org_id, + 'app_id': self.app_id, + 'collection_name': self.collection_name, + }) + + def entity(self, uuid): + pass + + def entity_from_data(self, data): + return UsergridEntity(org_id=self.org_id, + app_id=self.app_id, + collection_name=self.collection_name, + data=data, + client=self.client) + + def query(self, ql='select *', limit=100): + url = collection_query_url_template.format(app_id=self.app_id, + ql=ql, + limit=limit, + collection=self.collection_name, + **self.client.url_data) + + return UsergridQuery(url, headers=self.client.headers) + + def entities(self, **kwargs): + return self.query(**kwargs) + + def post(self, entity, **kwargs): + url = collection_url_template.format(collection=self.collection_name, + app_id=self.app_id, + **self.client.url_data) + + r = self.client.post(url, data=entity, **kwargs) + + if r.status_code == 200: + api_response = r.json() + entity = api_response.get('entities')[0] + e = UsergridEntity(org_id=self.org_id, + app_id=self.app_id, + collection_name=self.collection_name, + data=entity, + client=self.client) + return e + + else: + raise UsergridError(message='Unable to post to collection name=[%s]' % self.collection_name, + status_code=r.status_code, + data=entity, + api_response=r, + url=url) + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridConnection.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridConnection.py b/sdks/python/usergrid/UsergridConnection.py new file mode 100644 index 0000000..82d3fdc --- /dev/null +++ b/sdks/python/usergrid/UsergridConnection.py @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +import logging + + +class UsergridConnection(object): + def __init__(self, source_entity, verb, target_entity): + self.source_entity = source_entity + self.verb = verb + self.target_entity = target_entity + self.logger = logging.getLogger('usergrid.UsergridConnection') http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridError.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridError.py b/sdks/python/usergrid/UsergridError.py new file mode 100644 index 0000000..a5cf0bb --- /dev/null +++ b/sdks/python/usergrid/UsergridError.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridOrganization.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridOrganization.py b/sdks/python/usergrid/UsergridOrganization.py new file mode 100644 index 0000000..c0d345b --- /dev/null +++ b/sdks/python/usergrid/UsergridOrganization.py @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +from usergrid import UsergridApplication + + +class UsergridOrganization(object): + def __init__(self, org_id, client): + self.org_id = org_id + self.client = client + + def application(self, app_id): + return UsergridApplication(app_id, client=self.client) + + def app(self, app_id): + return self.application(app_id) + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridQueryIterator.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/UsergridQueryIterator.py b/sdks/python/usergrid/UsergridQueryIterator.py new file mode 100755 index 0000000..301ea7d --- /dev/null +++ b/sdks/python/usergrid/UsergridQueryIterator.py @@ -0,0 +1,155 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +import json +import logging +import traceback +import requests +import time + +__author__ = 'Jeff West @ ApigeeCorporation' + + +class UsergridQueryIterator(object): + def __init__(self, + url, + operation='GET', + sleep_time=10, + page_delay=0, + headers=None, + data=None): + + if not data: + data = {} + if not headers: + headers = {} + + self.page_counter = 0 + self.total_retrieved = 0 + self.logger = logging.getLogger('usergrid.UsergridQuery') + self.data = data + self.headers = headers + self.url = url + self.operation = operation + self.next_cursor = None + self.entities = [] + self.count_retrieved = 0 + self._pos = 0 + self.last_response = None + self.page_delay = page_delay + self.sleep_time = sleep_time + self.session = None + + def _get_next_response(self, attempts=0): + + if self.session is None: + self.session = requests.Session() + + try: + if self.operation == 'PUT': + op = self.session.put + elif self.operation == 'DELETE': + op = self.session.delete + else: + op = self.session.get + + target_url = self.url + + if self.next_cursor is not None: + delim = '&' if '?' in target_url else '?' + target_url = '%s%scursor=%s' % (self.url, delim, self.next_cursor) + + self.logger.debug('Operation=[%s] URL=[%s]' % (self.operation, target_url)) + + r = op(target_url, data=json.dumps(self.data), headers=self.headers) + + if r.status_code == 200: + r_json = r.json() + count_retrieved = len(r_json.get('entities', [])) + self.total_retrieved += count_retrieved + self.logger.debug('Retrieved [%s] entities in [%s]th page in [%s], total from [%s] is [%s]' % ( + count_retrieved, self.page_counter, r.elapsed, self.url, self.total_retrieved)) + + return r_json + + elif r.status_code in [401, 404] and 'service_resource_not_found' in r.text: + self.logger.error('Query Not Found [%s] on URL=[%s]: %s' % (r.status_code, target_url, r.text)) + raise SystemError('Query Not Found [%s] on URL=[%s]: %s' % (r.status_code, target_url, r.text)) + + else: + if attempts < 10: + self.logger.error('Sleeping %s after HTTP [%s] for retry attempt=[%s] on URL=[%s], response: %s' % ( + self.sleep_time, r.status_code, attempts, target_url, r.text)) + + time.sleep(self.sleep_time) + + return self._get_next_response(attempts=attempts + 1) + + else: + raise SystemError('Unable to get next response after %s attempts' % attempts) + + except: + print traceback.format_exc() + + def next(self): + + if self.last_response is None: + self.logger.debug('getting first page, url=[%s]' % self.url) + + self._process_next_page() + + elif self._pos >= len(self.entities) > 0 and self.next_cursor is not None: + + self.logger.debug('getting next page, count=[%s] url=[%s], cursor=[%s]' % ( + self.count_retrieved, self.url, self.next_cursor)) + + self._process_next_page() + self.logger.debug('Sleeping [%s]s between pages' % self.page_delay) + + time.sleep(self.page_delay) + + if self._pos < len(self.entities): + response = self.entities[self._pos] + self._pos += 1 + return response + + raise StopIteration + + def __iter__(self): + return self + + def _process_next_page(self, attempts=0): + + api_response = self._get_next_response() + + if api_response is None: + message = 'Unable to retrieve query results from url=[%s]' % self.url + self.logger.error(message) + api_response = {} + raise StopIteration + + self.last_response = api_response + + self.entities = api_response.get('entities', []) + self.next_cursor = api_response.get('cursor', None) + self._pos = 0 + self.count_retrieved += len(self.entities) + self.page_counter += 1 + + if self.next_cursor is None: + self.logger.debug('no cursor in response. Total pages=[%s], entities=[%s] url=[%s]' % ( + self.page_counter, self.count_retrieved, self.url)) http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/__init__.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/__init__.py b/sdks/python/usergrid/__init__.py new file mode 100644 index 0000000..93f8273 --- /dev/null +++ b/sdks/python/usergrid/__init__.py @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +__all__ = [ + 'UsergridApplication', + 'UsergridClient', + 'UsergridConnection', + 'UsergridConnectionProfile', + 'UsergridEntity', + 'Usergrid', + 'UsergridError', + 'UsergridOrganization', + 'UsergridAuth', + 'UsergridQueryIterator', + 'UsergridResponse' +] + +from .UsergridApplication import UsergridApplication +from .UsergridClient import UsergridClient, Usergrid, UsergridResponse +from .UsergridConnection import UsergridConnection +from .UsergridOrganization import UsergridOrganization +from .UsergridQueryIterator import UsergridQueryIterator +from .UsergridAuth import UsergridAuth http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/app_templates.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/app_templates.py b/sdks/python/usergrid/app_templates.py new file mode 100644 index 0000000..cb953ac --- /dev/null +++ b/sdks/python/usergrid/app_templates.py @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +__author__ = 'Jeff West @ ApigeeCorporation' + +org_url_template = "{base_url}/{org_id}" +app_url_template = "%s/{app_id}" % org_url_template + +app_token_url_template = "%s/token" % app_url_template + +collection_url_template = "%s/{collection}" % app_url_template +collection_query_url_template = "%s?ql={ql}&limit={limit}" % collection_url_template + +post_collection_url_template = collection_url_template +entity_url_template = "%s/{uuid_name}" % collection_url_template +get_entity_url_template = "%s?connections={connections}" % entity_url_template +put_entity_url_template = entity_url_template +delete_entity_url_template = entity_url_template + +assign_role_url_template = '%s/roles/{role_uuid_name}/{entity_type}/{entity_uuid_name}' % app_url_template + +connect_entities_by_type_template = '%s/{from_collection}/{from_uuid_name}/{relationship}/{to_collection}/{to_uuid_name}' % app_url_template \ No newline at end of file http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/management_templates.py ---------------------------------------------------------------------- diff --git a/sdks/python/usergrid/management_templates.py b/sdks/python/usergrid/management_templates.py new file mode 100644 index 0000000..62c531c --- /dev/null +++ b/sdks/python/usergrid/management_templates.py @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. The ASF licenses this file to You +# under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. For additional information regarding +# copyright in this work, please see the NOTICE file in the top level +# directory of this distribution. + +__author__ = 'Jeff West @ ApigeeCorporation' + +management_base_url = '{base_url}/management' +management_org_url_template = "%s/organizations/{org_id}" % management_base_url +management_org_list_apps_url_template = "%s/applications" % management_org_url_template +management_app_url_template = "%s/applications/{app_id}" % management_org_url_template + +org_token_url_template = "%s/token" % management_base_url http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/.gitignore ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/.gitignore b/utils/usergrid-util-python/.gitignore new file mode 100644 index 0000000..a6e3315 --- /dev/null +++ b/utils/usergrid-util-python/.gitignore @@ -0,0 +1,61 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# custom +sandbox +0e4b82c5-9aad-45de-810a-ff07c281ed2d_1454177649_export.zip http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/LICENSE ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/LICENSE b/utils/usergrid-util-python/LICENSE new file mode 100644 index 0000000..4db993e --- /dev/null +++ b/utils/usergrid-util-python/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Jeffrey West + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/README.md ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/README.md b/utils/usergrid-util-python/README.md new file mode 100644 index 0000000..7d3b533 --- /dev/null +++ b/utils/usergrid-util-python/README.md @@ -0,0 +1,15 @@ +# Usergrid Tools (in Python) + +## Prerequisites + +* Install the Usergrid Python SDK: `pip install usergrid` +* Install Usergrid Tools: `pip install usergrid-tools` + + +## Overview +The purpose of this module is to provide tools for working with Usergrid. The tools included as console scripts are: +* `usergrid_data_migrator` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/migration/README.md) A tool for migrating data from one Usergrid installation to another (or org1->org2) +* `parse_importer` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/parse_importer/README.md) A tool for importing data from a Parse.com data export into Usergrid +* `index_test` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/indexing/README.md) A tool for testing indexing latency in Usergrid + +For information on those tools, please see the respective README files http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/alias_mover.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py new file mode 100644 index 0000000..2a8fe02 --- /dev/null +++ b/utils/usergrid-util-python/es_tools/alias_mover.py @@ -0,0 +1,129 @@ +import json + +import requests + +example_request = { + "actions": [ + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias" + } + }, + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias" + } + }, + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias" + } + }, + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias" + } + }, + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias" + } + }, + { + "remove": { + "index": "apigee-vfmplus", + "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias" + } + }, + { + "add": { + "index": "apigee-vmplus-docvalues", + "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias" + } + } + ] +} + +cluster = 'rug000sr_euwi' + +work = { + # 'remove': { + # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3', + # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3', + # }, + 'add': { + '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18', + '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18', + } +} + +actions = [] + +for app_id, index in work.get('remove', {}).iteritems(): + actions.append({ + "remove": { + "index": index, + "alias": "%s_%s_read_alias" % (cluster, app_id) + }, + }) + actions.append({ + "remove": { + "index": index, + "alias": "%s_%s_write_alias" % (cluster, app_id) + }, + }) + +for app_id, index in work['add'].iteritems(): + actions.append({ + "add": { + "index": index, + "alias": "%s_%s_read_alias" % (cluster, app_id) + }, + }) + actions.append({ + "add": { + "index": index, + "alias": "%s_%s_write_alias" % (cluster, app_id) + }, + }) + +url = 'http://localhost:9200/_aliases' + +r = requests.post(url, data=json.dumps({'actions': actions})) + +print '%s: %s' % (r.status_code, r.text) http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py new file mode 100644 index 0000000..a462124 --- /dev/null +++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py @@ -0,0 +1,89 @@ +import json +import time +import requests + +__author__ = 'Jeff West @ ApigeeCorporation' + +# The purpose of this script is to set certain nodes in an ElasticSearch cluster to be excluded from index allocation, +# generally for the purpose of shutting down or restarting the node + +SHUTDOWN_NODES = True + +nodes = [ + # 'res206wo', + # 'res207wo', +] + +base_url = 'http://localhost:9200' + +exclude_nodes = nodes + +nodes_string = ",".join(exclude_nodes) + +print 'Excluding: ' + nodes_string +url_template = '%s/_cluster/settings' % base_url + +status_code = 503 + +while status_code >= 500: + r = requests.put( + '%s/_cluster/settings' % base_url, + data=json.dumps({ + "transient": { + "cluster.routing.allocation.exclude._host": nodes_string + } + })) + + status_code = r.status_code + + print '%s: %s' % (r.status_code, r.text) + +ready = False + +nodes_shut_down = [] + +while not ready: + + ready = True + nodes_left = 0 + bytes_left = 0 + + for node in exclude_nodes: + node_url = '%s/_nodes/%s/stats' % (base_url, node) + r = requests.get(node_url) + + if r.status_code == 200: + # print r.text + + node_stats = r.json() + + for field, data in node_stats.get('nodes').iteritems(): + if data.get('name') == node: + size = data.get('indices', {}).get('store', {}).get('size_in_bytes', 1) + docs = data.get('indices', {}).get('docs', {}).get('count', 1) + + if size > 0 and docs > 0: + print 'Node: %s - size %s' % (node, size) + bytes_left += size + ready = False and ready + nodes_left += 1 + else: + if SHUTDOWN_NODES: + if not node in nodes_shut_down: + nodes_shut_down.append(node) + shutdown_url = '%s/_cluster/nodes/%s/_shutdown' % (base_url, node) + + print 'Shutting down node %s: %s' % (node, shutdown_url) + + r = requests.post(shutdown_url) + + if r.status_code == 200: + nodes_shut_down.append(node) + print 'Shut down node %s' % node + else: + print 'Shutdown failed: %s: %s' % (r.status_code, r.text) + if not ready: + print 'NOT READY! Waiting for %s nodes and %s GB' % (nodes_left, bytes_left / 1024.0 / 1000000) + time.sleep(10) + +# print 'READY TO MOVE!' http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/command_sender.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py new file mode 100644 index 0000000..92ecfff --- /dev/null +++ b/utils/usergrid-util-python/es_tools/command_sender.py @@ -0,0 +1,42 @@ +import json +import requests + +__author__ = 'Jeff West @ ApigeeCorporation' + + +# Simple utility to send commands, useful to not have to recall the proper format + +# +# url = 'http://localhost:9200/_cat/shards' +# +# r = requests.get(url) +# +# response = r.text +# +# print response + +data = { + "commands": [ + { + "move": { + "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final", + "shard": 14, + "from_node": "res018sy", + "to_node": "res021sy" + } + }, + { + "move": { + "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final", + "shard": 12, + "from_node": "res018sy", + "to_node": "res009sy" + } + }, + + ] +} + +r = requests.post('http://localhost:9211/_cluster/reroute', data=json.dumps(data)) + +print r.text \ No newline at end of file http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py new file mode 100644 index 0000000..f151fcb --- /dev/null +++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py @@ -0,0 +1,107 @@ +import json +import re +from multiprocessing.pool import Pool +import requests + +# This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity + +index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}' + +index_names = [ + 'es-index-name' +] + +baas_url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}' + +counter = 0 +size = 1000 + +total_docs = 167501577 +from_var = 0 +page = 0 + +work_items = [] + + +def work(item): + url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'.format( + app_id=item[0], + collection=item[1], + entity_id=item[2] + ) + + r_put = requests.put(url, data=json.dumps({'russo': ''})) + + if r_put.status_code == 200: + print '[%s]: %s' % (r_put.status_code, url) + + elif r_put.status_code: + print '[%s]: %s | %s' % (r_put.status_code, url, r.text) + + +while from_var < total_docs: + + from_var = page * size + page += 1 + + for index_name in index_names: + + index_url = index_url_template.format(index_name=index_name, size=size, from_var=from_var) + + print 'Getting URL: ' + index_url + + r = requests.get(index_url) + + if r.status_code != 200: + print r.text + exit() + + response = r.json() + + hits = response.get('hits', {}).get('hits') + + re_app_id = re.compile('appId\((.+),') + re_ent_id = re.compile('entityId\((.+),') + re_type = re.compile('entityId\(.+,(.+)\)') + + print 'Index: %s | hits: %s' % (index_name, len(hits)) + + for hit_data in hits: + source = hit_data.get('_source') + + application_id = source.get('applicationId') + + app_id_find = re_app_id.findall(application_id) + + if len(app_id_find) > 0: + app_id = app_id_find[0] + + if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc': + continue + + entity_id_tmp = source.get('entityId') + + entity_id_find = re_ent_id.findall(entity_id_tmp) + entity_type_find = re_type.findall(entity_id_tmp) + + if len(entity_id_find) > 0 and len(entity_type_find) > 0: + entity_id = entity_id_find[0] + collection = entity_type_find[0] + + if collection in ['logs', 'log']: + print 'skipping logs...' + continue + + work_items.append((app_id, collection, entity_id)) + + counter += 1 + +pool = Pool(16) + +print 'Work Items: %s' % len(work_items) + +print 'Starting Work' + +pool.map(work, work_items) + +print 'done: %s' % counter http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/es_searcher.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py new file mode 100644 index 0000000..55e54ef --- /dev/null +++ b/utils/usergrid-util-python/es_tools/es_searcher.py @@ -0,0 +1,24 @@ +import json +import requests + +# Simple example of searching for a specific entity in ES + +__author__ = 'Jeff West @ ApigeeCorporation' + +url_template = 'http://localhost:9200/pea000ug_applications_2/_search' + +request = { + "query": { + "term": { + "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)" + } + } +} + +# url_template = 'http://localhost:9200/_search' +# r = requests.get(url) +r = requests.get(url_template, data=json.dumps(request)) + +print r.status_code +print json.dumps(r.json(), indent=2) + http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_deleter.py ---------------------------------------------------------------------- diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py new file mode 100644 index 0000000..a697cf8 --- /dev/null +++ b/utils/usergrid-util-python/es_tools/index_deleter.py @@ -0,0 +1,68 @@ +import requests +import logging + +__author__ = 'Jeff West @ ApigeeCorporation' + + +# utility for deleting indexes that are no longer needed. Given: +# A) a set of strings to include when evaluating the index names to delete +# B) a set of strings to Exclude when evaluating the index names to delete +# +# The general logic is: +# 1) If the include set is empty, or if the index name contains a string in the 'include' set, then delete +# 2) If the index contains a string in the exclude list, do not delete + +url_base = 'http://localhost:9200' + +r = requests.get(url_base + "/_stats") + +indices = r.json()['indices'] + +print 'retrieved %s indices' % len(indices) + +NUMBER_VALUE = 0 + +includes = [ + 'rug002sr_euwi', + # 'rug002mr', + # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3', + # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9', +] + +excludes = [ + # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8', + # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3', + # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9', + # 'a34ad389-b626-11e4-848f-06b49118d7d0' +] + +counter = 0 +process = False +delete_counter = 0 + +for index in indices: + process = False + counter += 1 + + print 'index %s of %s' % (counter, len(indices)) + + if len(includes) == 0: + process = True + else: + for include in includes: + + if include in index: + process = True + + if len(excludes) > 0: + for exclude in excludes: + if exclude in index: + process = False + + if process: + delete_counter += 1 + + url_template = '%s/%s' % (url_base, index) + print 'DELETING Index [%s] %s at URL %s' % (delete_counter, index, url_template) + + response = requests.delete('%s/%s' % (url_base, index))
