anime-dl
/
anime-downloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed unneeded imports

master
Arjix 2020-09-02 18:31:25 +03:00 committed by GitHub
parent 91870487a9
commit 8cb3a8dfb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 36 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
anime_downloader/sites/helpers/selescrape.py
View File

@ -1,15 +1,8 @@
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.remote.remote_connection import LOGGER as serverLogger
from selenium.webdriver.support.ui import WebDriverWait
from anime_downloader.const import get_random_header
from selenium.webdriver.common.by import By
from urllib.parse import urlencode
from urllib.parse import urlsplit
from selenium import webdriver
from bs4 import BeautifulSoup
from logging import exception
from sys import platform
import requests
import tempfile
import os
import logging
@ -20,6 +13,7 @@ import json
serverLogger.setLevel(logging.ERROR)
logger = logging.getLogger(__name__)
def get_data_dir():
'''
Gets the folder directory selescrape will store data,
@ -33,6 +27,7 @@ def open_config():
from anime_downloader.config import Config
return Config
data = open_config()
@ -40,20 +35,23 @@ def get_browser_config():
'''
Decides what browser selescrape will use.
'''
os_browser = { #maps os to a browser
'linux':'firefox',
'darwin':'chrome',
'win32':'chrome'
os_browser = { # maps os to a browser
'linux': 'firefox',
'darwin': 'chrome',
'win32': 'chrome'
}
for a in os_browser:
if platform.startswith(a):
browser = os_browser[a]
browser = os_browser[a]
else:
browser = 'chrome'
value = data['dl']['selescrape_browser']
value = value.lower() if value else value
if value in ['chrome', 'firefox']:
browser = value
return browser
@ -88,11 +86,12 @@ def cache_request(sele_response):
'method': data['method'],
'cookies': data['cookies'],
'user_agent': data['user_agent']
}
}
with open(file, 'w') as f:
json.dump(tmp_cache, f, indent=4)
def check_cache(url):
"""
This function checks if the cache file exists,
@ -136,27 +135,28 @@ def driver_select():
fireFox_Profile = webdriver.FirefoxProfile()
fireFox_Profile.set_preference("general.useragent.override", get_random_header()['user-agent'])
if binary == None:
if not binary:
driver = webdriver.Firefox(fireFox_Profile, options=fireFox_Options, service_log_path=os.path.devnull)
else:
try:
driver = webdriver.Firefox(fireFox_Profile, options=fireFox_Options, service_log_path=os.path.devnull)
except:
driver = webdriver.Firefox(fireFox_Profile, executable_path=binary, options=fireFox_Options, service_log_path=os.path.devnull)
driver = webdriver.Firefox(fireFox_Profile, executable_path=binary, options=fireFox_Options,
service_log_path=os.path.devnull)
elif browser == 'chrome':
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
ops = ["--headless", "--disable-gpu", '--log-level=OFF', f"--user-data-dir={profile_path}",
"--no-sandbox", "--window-size=1920,1080", f"user-agent={get_random_header()['user-agent']}"]
"--no-sandbox", "--window-size=1920,1080", f"user-agent={get_random_header()['user-agent']}"]
for option in ops:
chrome_options.add_argument(option)
profile_path = os.path.join(data_dir, 'Selenium_chromium')
log_path = os.path.join(data_dir, 'chromedriver.log')
if binary == None:
if executable == None:
if not binary:
if not executable:
driver = webdriver.Chrome(options=chrome_options)
else:
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
@ -164,13 +164,14 @@ def driver_select():
cap['binary_location'] = executable
driver = webdriver.Chrome(desired_capabilities=cap, options=chrome_options)
else:
if executable == None:
if not executable:
driver = webdriver.Chrome(options=chrome_options)
else:
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
cap = DesiredCapabilities.CHROME
cap['binary_location'] = executable
driver = webdriver.Chrome(executable_path=binary, desired_capabilities=cap, options=chrome_options, service_log_path=os.path.devnull)
driver = webdriver.Chrome(executable_path=binary, desired_capabilities=cap, options=chrome_options,
service_log_path=os.path.devnull)
return driver
@ -199,10 +200,10 @@ def cloudflare_wait(driver):
title = driver.title
if not title == "Just a moment...":
break
time.sleep(1) # This is necessary to make sure everything has loaded fine.
time.sleep(1) # This is necessary to make sure everything has loaded fine.
def request(request_type, url, **kwargs): #Headers not yet supported , headers={}
def request(request_type, url, **kwargs): # Headers not yet supported , headers={}
params = kwargs.get('params', {})
url = url if not params else url + '?' + urlencode(params)
@ -222,7 +223,7 @@ def request(request_type, url, **kwargs): #Headers not yet supported , headers={
try:
cloudflare_wait(driver)
user_agent = driver.execute_script("return navigator.userAgent;") #dirty, but allows for all sorts of things above
user_agent = driver.execute_script("return navigator.userAgent;")
cookies = driver.get_cookies()
text = driver.page_source
driver.close()
@ -235,11 +236,10 @@ def request(request_type, url, **kwargs): #Headers not yet supported , headers={
driver.save_screenshot(f"{get_data_dir()}/screenshot.png");
driver.close()
logger.error(f'There was a problem getting the page: {url}.' +
'\nSee the screenshot for more info:\t{get_data_dir()}/screenshot.png')
'\nSee the screenshot for more info:\t{get_data_dir()}/screenshot.png')
exit()
class SeleResponse:
"""
Class for the selenium response.
@ -257,6 +257,7 @@ class SeleResponse:
user_agent: string
User agent used on the webpage
"""
def __init__(self, url, method, text, cookies, user_agent):
self.url = url
self.method = method