(.+?)',re.DOTALL).findall(t)
summary = n[0]
summary = clean_search(summary)
summary = remove_non_ascii(summary)
summary = summary.encode('utf8')
xml += "- "\
"%s"\
""\
"movie"\
""\
""\
""\
"%s"\
"%s"\
"%s"\
""\
"link**%s**%s**%s"\
"
" % (name,image,image,summary,link,name,image)
except:
pass
next_page = int(current)+1
xml += "- "\
"[COLOR dodgerblue]Next Page >>[/COLOR]"\
"tvshow/%s"\
"http://www.clker.com/cliparts/a/f/2/d/1298026466992020846arrow-hi.png"\
"
" % (next_page)
jenlist = JenList(xml)
display_list(jenlist.get_list(), jenlist.get_content_type(), pins)
@route(mode='get_metacritic_trailer_link', args=["url"])
def get_game(url):
try:
koding.Show_Busy(status=True)
link1 = url.split("**")[-3]
name = url.split("**")[-2]
thumbnail = url.split("**")[-1]
t = scraper.get(link1).content
m2 = re.compile('.+?data-mcvideourl="(.+?)"',re.DOTALL).findall(t)
final_link = m2[0]
koding.Show_Busy(status=False )
info = xbmcgui.ListItem(name, thumbnailImage=thumbnail)
xbmc.Player().play(final_link,info)
except:
pass
def fetch_from_db2(url):
koding.reset_db()
url2 = clean_url(url)
match = koding.Get_All_From_Table(url2)
if match:
match = match[0]
if not match["value"]:
return None
match_item = match["value"]
try:
result = pickle.loads(base64.b64decode(match_item))
except:
return None
created_time = match["created"]
print created_time + "created"
print time.time()
print CACHE_TIME
test_time = float(created_time) + CACHE_TIME
print test_time
if float(created_time) + CACHE_TIME <= time.time():
koding.Remove_Table(url2)
db = sqlite3.connect('%s' % (database_loc))
cursor = db.cursor()
db.execute("vacuum")
db.commit()
db.close()
display_list2(result, "video", url2)
else:
pass
return result
else:
return []
def remove_non_ascii(text):
return unidecode(text)
def clean_search(title):
if title == None: return
title = re.sub('&#(\d+);', '', title)
title = re.sub('(&#[0-9]+)([^;^0-9]+)', '\\1;\\2', title)
title = title.replace('"', '\"').replace('&', '&').replace("'","")
title = re.sub('\\\|/|\(|\)|\[|\]|\{|\}|-|:|;|\*|\?|"|\'|<|>|\_|\.|\?', ' ', title)
title = ' '.join(title.split())
return title
class CloudflareAdapter(HTTPAdapter):
""" HTTPS adapter that creates a SSL context with custom ciphers """
def get_connection(self, *args, **kwargs):
conn = super(CloudflareAdapter, self).get_connection(*args, **kwargs)
if conn.conn_kw.get("ssl_context"):
conn.conn_kw["ssl_context"].set_ciphers(DEFAULT_CIPHERS)
else:
context = create_urllib3_context(ciphers=DEFAULT_CIPHERS)
conn.conn_kw["ssl_context"] = context
return conn
class CloudflareScraper(Session):
def __init__(self, *args, **kwargs):
self.tries = 0
self.prev_resp = None
self.delay = kwargs.pop("delay", None)
# Use headers with a random User-Agent if no custom headers have been set
headers = OrderedDict(kwargs.pop("headers", DEFAULT_HEADERS))
# Set the User-Agent header if it was not provided
headers.setdefault("User-Agent", DEFAULT_USER_AGENT)
super(CloudflareScraper, self).__init__(*args, **kwargs)
# Define headers to force using an OrderedDict and preserve header order
self.headers = headers
self.mount("https://", CloudflareAdapter())
@staticmethod
def is_cloudflare_iuam_challenge(resp, allow_empty_body=False):
return (
resp.status_code in (503, 429)
and resp.headers.get("Server", "").startswith("cloudflare")
and (allow_empty_body or (b"jschl_vc" in resp.content and b"jschl_answer" in resp.content))
)
@staticmethod
def is_cloudflare_captcha_challenge(resp):
return (
resp.status_code == 403
and resp.headers.get("Server", "").startswith("cloudflare")
and b"/cdn-cgi/l/chk_captcha" in resp.content
)
def request(self, method, url, *args, **kwargs):
resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
# Check if Cloudflare captcha challenge is presented
if self.is_cloudflare_captcha_challenge(resp):
self.handle_captcha_challenge()
self.prev_resp = resp
# Check if Cloudflare anti-bot "I'm Under Attack Mode" is enabled
if self.is_cloudflare_iuam_challenge(resp):
if self.tries >= 3:
exception_message = 'Failed to solve Cloudflare challenge!'
if os.getenv('CI') == 'true':
exception_message += '\n' + resp.text
raise Exception(exception_message)
resp = self.solve_cf_challenge(resp, **kwargs)
return resp
def cloudflare_is_bypassed(self, url, resp=None):
cookie_domain = ".{}".format(urlparse(url).netloc)
return (
self.cookies.get("cf_clearance", None, domain=cookie_domain) or
(resp and resp.cookies.get("cf_clearance", None, domain=cookie_domain))
)
def handle_captcha_challenge(self):
exception_message = 'Cloudflare returned captcha!'
if self.prev_resp is not None and os.getenv('CI') == 'true':
exception_message += '\n' + self.prev_resp.text
raise Exception(exception_message)
def solve_cf_challenge(self, resp, **original_kwargs):
self.tries += 1
start_time = time.time()
body = resp.text
parsed_url = urlparse(resp.url)
domain = parsed_url.netloc
submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain)
cloudflare_kwargs = copy.deepcopy(original_kwargs)
headers = cloudflare_kwargs.setdefault("headers", {})
headers["Referer"] = resp.url
try:
params = cloudflare_kwargs["params"] = OrderedDict(
re.findall(r'name="(s|jschl_vc|pass)"(?: [^<>]*)? value="(.+?)"', body)
)
for k in ("jschl_vc", "pass"):
if k not in params:
raise ValueError("%s is missing from challenge form" % k)
except Exception as e:
# Something is wrong with the page.
# This may indicate Cloudflare has changed their anti-bot
# technique. If you see this and are running the latest version,
# please open a GitHub issue so I can update the code accordingly.
raise ValueError(
"Unable to parse Cloudflare anti-bot IUAM page: %s %s"
% (e.message, BUG_REPORT)
)
# Solve the Javascript challenge
answer, delay = solve_challenge(body, domain)
params["jschl_answer"] = answer
# Requests transforms any request into a GET after a redirect,
# so the redirect has to be handled manually here to allow for
# performing other types of requests even as the first request.
method = resp.request.method
cloudflare_kwargs["allow_redirects"] = False
# Cloudflare requires a delay before solving the challenge
if not self.delay:
time.sleep(max(delay - (time.time() - start_time), 0))
else:
time.sleep(self.delay)
# Send the challenge response and handle the redirect manually
redirect = self.request(method, submit_url, **cloudflare_kwargs)
redirect_location = urlparse(redirect.headers["Location"])
if not redirect_location.netloc:
redirect_url = urlunparse(
(
parsed_url.scheme,
domain,
redirect_location.path,
redirect_location.params,
redirect_location.query,
redirect_location.fragment,
)
)
return self.request(method, redirect_url, **original_kwargs)
return self.request(method, redirect.headers["Location"], **original_kwargs)
create_scraper = CloudflareScraper
scraper = create_scraper()