⬆️ Changed to my_fake_useragent

Author: officialpm

scrape_amazon/util/urlFunctions.py

@@ -1,11 +1,12 @@
 import requests
+from my_fake_useragent import UserAgent
 
 url_prefix = "https://www.amazon"
-headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept-Encoding":"gzip, deflate", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
+
 # random user-agent
+ua = UserAgent(family="chrome", os_family="windows")
+# ua = UserAgent(cache=False, use_cache_server=False, safe_attrs=("__injections__",))
 
-from fake_useragent import UserAgent
-ua = UserAgent(cache=False, use_cache_server=False)
 
 def construst_reviews_URL(domain: str, product_id: str) -> str:
  """Constructs review URL.
@@ -16,19 +17,20 @@ def construst_reviews_URL(domain: str, product_id: str) -> str:
  """
  return f"{url_prefix}.{domain}/dp/product-reviews/{product_id}"
 
+
 def get_URL(url: str) -> str:
  """Gets the contents of a remote url.
  Args:
  url
  Returns:
  The content fetched from remote url.
  """
- user_agent = ua.random
- while (True):
- content: str = requests.get(url, headers={'User-Agent': user_agent})
+ user_agent = ua.random()
+ while True:
+ content: str = requests.get(url, headers={"User-Agent": user_agent})
  if "api-services-support@amazon.com" in content.text:
- user_agent = ua.random
+ user_agent = ua.random()
  continue
  break
- 
- return content
+
+ return content

setup.py

@@ -1,33 +1,35 @@
 import setuptools
 
-with open('README.md') as readme_file:
+with open("README.md") as readme_file:
  README = readme_file.read()
 
-setuptools.setup(name='scrape_amazon',
- version='0.1.4',
- description='Scrape Amazon Reviews',
- url='http://github.com/officialpm/scrape-amazon',
- author='Parth Maniar',
- author_email='officialparthmaniar@gmail.com',
- license='MIT',
- long_description_content_type="text/markdown",
- long_description=README,
- download_url="https://pypi.org/project/scrape-amazon",
- packages=setuptools.find_packages(),
- keywords=["Amazon", "Scrape", "Reviews", "Scraper", "Products"],
- python_requires='>=3.6',
- classifiers=[
- "Operating System :: OS Independent",
- "Intended Audience :: Developers",
- "Topic :: Utilities",
- "Natural Language :: English",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3 :: Only"
- ], install_requires=['beautifulsoup4', 'pandas', 'p_tqdm', 'fake_useragent'],
- entry_points={
- 'console_scripts': ['scrape-amazon=scrape_amazon.cli:get_reviews_cli'],
- }
- )
+setuptools.setup(
+ name="scrape_amazon",
+ version="0.1.4",
+ description="Scrape Amazon Reviews",
+ url="http://github.com/officialpm/scrape-amazon",
+ author="Parth Maniar",
+ author_email="officialparthmaniar@gmail.com",
+ license="MIT",
+ long_description_content_type="text/markdown",
+ long_description=README,
+ download_url="https://pypi.org/project/scrape-amazon",
+ packages=setuptools.find_packages(),
+ keywords=["Amazon", "Scrape", "Reviews", "Scraper", "Products"],
+ python_requires=">=3.6",
+ classifiers=[
+ "Operating System :: OS Independent",
+ "Intended Audience :: Developers",
+ "Topic :: Utilities",
+ "Natural Language :: English",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3 :: Only",
+ ],
+ install_requires=["beautifulsoup4", "pandas", "p_tqdm", "my_fake_useragent"],
+ entry_points={
+ "console_scripts": ["scrape-amazon=scrape_amazon.cli:get_reviews_cli"],
+ },
+)