what is the python version ? Generally, It means the value of data is
None. You can simply print "enumerate(search_items, start=1)" to confirm
the value type.
On Saturday, May 8, 2021 at 9:10:12 AM UTC-4 amol.s...@namasys.co wrote:
> output:
> result = self.fn(*self.args, **self.kwargs)
> File "app-1.py", line 103, in run
> results.append(self.find_keyword(keyword, country))
> File "app-1.py", line 75, in find_keyword
> for i, search_item in enumerate(search_items, start=1):
> TypeError: 'NoneType' object is not iterable
>
>
>
>
>
>
>
> my code:
>
> import undetected_chromedriver as uc
> uc.install()
> import sys, os
> import traceback
> import requests
> import urllib.parse as p
> import pandas as pd
> import datetime
> from bs4 import BeautifulSoup
> import random
> from selenium import webdriver
> from selenium.webdriver.common.by import By
> from selenium.webdriver.common.keys import Keys
> from selenium.webdriver.chrome.options import Options
> from time import sleep
> import time
> import re
> import threading
> import concurrent.futures
> import csv
> from config import *
> import math
> import code
>
>
>
>
> date = datetime.datetime.now()
> date_format = date.strftime("%d-%m-%Y, %H:%M:%S")
> date_only = date.strftime("%d-%m-%Y-%H%M%S")
>
> import logging
> import sys
>
> logger = logging.getLogger()
> logger.setLevel(logging.INFO)
> formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s',
> '%m-%d-%Y %H:%M:%S')
> file_handler = logging.FileHandler(f'logs/{date_only}.log',
> encoding="utf8")
> file_handler.setLevel(logging.DEBUG)
> file_handler.setFormatter(formatter)
>
> logger.addHandler(file_handler)
>
> class Process:
> def __init__(self, keywords, url):
> self.keywords = keywords
> #self.no_of_pages = no_of_pages
> self.count = 1
> self.url = url
> self.driver = None
>
>
> def find_keyword(self, keyword,country):
>
>
> for page in range(1, 11):
>
> print("[*] Going for page:", page)
> # calculating start
> start = (page - 1) * 10 + 1
> # make API request
> url = f"
> https://www.googleapis.com/customsearch/v1?key={API_KEY}&cx={SEARCH_ENGINE_ID}&q={keyword}&gl={country}&start={start}
> "
>
> data = requests.get(url).json()
>
> search_items = data.get("items")
>
> # a boolean that indicates whether `target_domain` is found
> found = False
> for i, search_item in enumerate(search_items, start=1):
> # get the page title
> title = search_item.get("title")
> # page snippet
> snippet = search_item.get("snippet")
> # alternatively, you can get the HTML snippet (bolded
> keywords)
> html_snippet = search_item.get("htmlSnippet")
> # extract the page url
> link = search_item.get("link")
> # extract the domain name from the URL
> domain_name = p.urlparse(link).netloc
> if domain_name.endswith(self.url):
> # get the page rank
> rank = i + start - 1
> print(f"[+] {self.url} is found on rank #{rank} for
> keyword: '{keyword}'")
> print("[+] Title:", title)
> print("[+] Snippet:", snippet)
> print("[+] URL:", link)
> logging.info(f"link -- {link}")
> return {'keyword': keyword, 'rank' : rank}
> # target domain is found, exit out of the program
> else:
> return {'keyword': keyword, 'rank' : 0}
>
> def run(self):
> results = []
>
> for keyword, country in self.keywords:
> results.append(self.find_keyword(keyword, country))
>
> return results
>
> if __name__=="__main__":
> result = []
> df = pd.read_csv('Input/Keywords - Sheet2.csv')
> keywords = list(df['keyword'].values)
> countries = list(df['country'].values)
> kc = list(zip(keywords,countries))
> url = "vpnbrains.com"
> row = len(keywords)
> arr = []
> batch = BATCH
> batchsize = math.ceil(row/batch)
>
> for i in range(batch):
> arr.append(kc[i*batchsize:(i+1)*batchsize])
>
>
> with concurrent.futures.ThreadPoolExecutor() as executor:
> futures = []
> for af in arr :
> if (i+1)*batchsize > row:
> obj = Process(kc,url)
> futures.append(
> executor.submit(
> obj.run
> )
> )
> else:
> obj = Process(af,url)
> futures.append(
>
> executor.submit(
> obj.run
> )
> )
> for j in futures:
> res = j.result()
> result.extend(res)
>
> out_df = pd.DataFrame(result)
> out_df.to_csv(f'Export/{date_only}.csv', index=False)
> code.interact(local=locals())
--
You received this message because you are subscribed to the Google Groups
"Google App Engine" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to google-appengine+unsubscr...@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/google-appengine/7a774213-8494-424e-96b1-83168d78079an%40googlegroups.com.
