{ "metadata": { "name": "" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "from ghost import Ghost\n", "from bs4 import BeautifulSoup as bs\n", "import regex\n", "import pandas\n", "ghost = Ghost().start()" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 1 }, { "cell_type": "code", "collapsed": false, "input": [ "page = ghost.open(\"http://www.whoscored.com/Teams/1232\")" ], "language": "python", "metadata": {}, "outputs": [ { "ename": "TimeoutError", "evalue": "Unable to load requested page", "output_type": "pyerr", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTimeoutError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mghost\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"http://www.whoscored.com/Teams/1232\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/ghost/ghost.pyc\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, address, method, headers, auth, body, default_popup_response, wait, timeout, client_certificate, encode_url, user_agent)\u001b[0m\n\u001b[1;32m 899\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 900\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 901\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_page_loaded\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 902\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mscroll_to_anchor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0manchor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/ghost/ghost.pyc\u001b[0m in \u001b[0;36mwait_for_page_loaded\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 1194\u001b[0m \"\"\"\n\u001b[1;32m 1195\u001b[0m self.wait_for(lambda: self.loaded,\n\u001b[0;32m-> 1196\u001b[0;31m 'Unable to load requested page', timeout)\n\u001b[0m\u001b[1;32m 1197\u001b[0m \u001b[0mresources\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_release_last_resources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1198\u001b[0m \u001b[0mpage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/ghost/ghost.pyc\u001b[0m in \u001b[0;36mwait_for\u001b[0;34m(self, condition, timeout_message, timeout)\u001b[0m\n\u001b[1;32m 1172\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcondition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1173\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mstarted_at\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1174\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1175\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1176\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_callback\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mTimeoutError\u001b[0m: Unable to load requested page" ] } ], "prompt_number": 5 }, { "cell_type": "code", "collapsed": false, "input": [ "soup = bs(ghost.content)" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 234 }, { "cell_type": "code", "collapsed": false, "input": [ "table = soup.find('tbody', { \"id\" : \"team-squad-stats-summary-content\" })" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 312 }, { "cell_type": "code", "collapsed": false, "input": [ "players = table.findAll('tr')" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 313 }, { "cell_type": "code", "collapsed": false, "input": [ "data_all = {}\n", "for player in players:\n", " cols = player.findAll('td')\n", " data = [col.get_text() for col in cols]\n", " data_all[data[2]] = data" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 314 }, { "cell_type": "code", "collapsed": false, "input": [ "pandas.DataFrame(data_all).T" ], "language": "python", "metadata": {}, "outputs": [ { "html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
012345678910111213141516
Biro Biro 12 Biro Biro AM(L) 19 164 55 0(1) - - - - 1 60 - - 6.47
Bruno Vieira 5 Bruno Vieira D(R) 28 178 69 2 - - - - - 85.9 1.5 - 7.23
Carlinhos 2 Carlinhos D(L) 27 176 72 2 - - - - 1 81.1 1 1 7.57
Chiquinho 17 Chiquinho AM(C) 24 170 69 - - - - - - - - - -
Dar\u00edo Conca 6 Dar\u00edo Conca M 30 167 58 2 - - 1 - 3 85.9 - - 7.21
Diego Cavalieri 10 Diego Cavalieri GK 31 191 86 2 - - - - - 46.2 - - 6.69
Diguinho 8 Diguinho DM(C) 31 171 70 2 - - - - 0.5 84.7 1.5 - 7.14
Edwin Valencia 13 Edwin Valencia M 29 181 85 0(2) - - - - - 81.8 1 - 6.31
Elivelton 9 Elivelton D 21 179 81 2 - - - - - 74.6 0.5 - 6.86
Felipe 16 Felipe GK 26 193 90 - - - - - - - - - -
Fred 7 Fred FW 30 185 75 2 1 2 - - 3 75.6 1.5 - 7.19
Gum 3 Gum D(C) 28 189 86 2 - - - - 0.5 75.2 2.5 1 7.51
Jean 11 Jean DM(C) 27 170 70 2 - - - - 1 88.6 - - 6.61
Kenedy 18 Kenedy FW 18 182 77 - - - - - - - - - -
Rafael S\u00f3bis 1 Rafael S\u00f3bis AM(LR),FW 28 172 67 2 2 - - - 3.5 78.8 1.5 - 7.82
Rafinha 14 Rafinha M(R) 20 177 74 0(1) - - - - - 50 - - 6
Wagner 4 Wagner AM(C) 29 172 71 2 - - 1 - 3 90 - - 7.43
Walter 15 Walter FW 24 178 88 0(1) - - - - - 60 - - 5.95
Wellington Carvalho 19 Wellington Carvalho D 21 185 84 - - - - - - - - - -
Wellington Silva 20 Wellington Silva D 26 177 64 - - - - - - - - - -
\n", "

20 rows \u00d7 17 columns

\n", "
" ], "metadata": {}, "output_type": "pyout", "prompt_number": 315, "text": [ " 0 1 2 3 4 5 6 7 \\\n", "Biro Biro 12 Biro Biro AM(L) 19 164 55 0(1) \n", "Bruno Vieira 5 Bruno Vieira D(R) 28 178 69 2 \n", "Carlinhos 2 Carlinhos D(L) 27 176 72 2 \n", "Chiquinho 17 Chiquinho AM(C) 24 170 69 - \n", "Dar\u00edo Conca 6 Dar\u00edo Conca M 30 167 58 2 \n", "Diego Cavalieri 10 Diego Cavalieri GK 31 191 86 2 \n", "Diguinho 8 Diguinho DM(C) 31 171 70 2 \n", "Edwin Valencia 13 Edwin Valencia M 29 181 85 0(2) \n", "Elivelton 9 Elivelton D 21 179 81 2 \n", "Felipe 16 Felipe GK 26 193 90 - \n", "Fred 7 Fred FW 30 185 75 2 \n", "Gum 3 Gum D(C) 28 189 86 2 \n", "Jean 11 Jean DM(C) 27 170 70 2 \n", "Kenedy 18 Kenedy FW 18 182 77 - \n", "Rafael S\u00f3bis 1 Rafael S\u00f3bis AM(LR),FW 28 172 67 2 \n", "Rafinha 14 Rafinha M(R) 20 177 74 0(1) \n", "Wagner 4 Wagner AM(C) 29 172 71 2 \n", "Walter 15 Walter FW 24 178 88 0(1) \n", "Wellington Carvalho 19 Wellington Carvalho D 21 185 84 - \n", "Wellington Silva 20 Wellington Silva D 26 177 64 - \n", "\n", " 8 9 10 11 12 13 14 15 16 \n", "Biro Biro - - - - 1 60 - - 6.47 \n", "Bruno Vieira - - - - - 85.9 1.5 - 7.23 \n", "Carlinhos - - - - 1 81.1 1 1 7.57 \n", "Chiquinho - - - - - - - - - \n", "Dar\u00edo Conca - - 1 - 3 85.9 - - 7.21 \n", "Diego Cavalieri - - - - - 46.2 - - 6.69 \n", "Diguinho - - - - 0.5 84.7 1.5 - 7.14 \n", "Edwin Valencia - - - - - 81.8 1 - 6.31 \n", "Elivelton - - - - - 74.6 0.5 - 6.86 \n", "Felipe - - - - - - - - - \n", "Fred 1 2 - - 3 75.6 1.5 - 7.19 \n", "Gum - - - - 0.5 75.2 2.5 1 7.51 \n", "Jean - - - - 1 88.6 - - 6.61 \n", "Kenedy - - - - - - - - - \n", "Rafael S\u00f3bis 2 - - - 3.5 78.8 1.5 - 7.82 \n", "Rafinha - - - - - 50 - - 6 \n", "Wagner - - 1 - 3 90 - - 7.43 \n", "Walter - - - - - 60 - - 5.95 \n", "Wellington Carvalho - - - - - - - - - \n", "Wellington Silva - - - - - - - - - \n", "\n", "[20 rows x 17 columns]" ] } ], "prompt_number": 315 } ], "metadata": {} } ] }