Commit 01ff7ab4 authored by Romain Courteaud's avatar Romain Courteaud

Add SNI support

parent 822ac1a4
# URL checker contribution # URL checker contribution
## Install development environnment ## Install development environnment
virtualenv venv virtualenv venv
. venv/bin/activate . venv/bin/activate
pip install black pyflakes pip install black pyflakes
pip install --upgrade --editable .
## Check the code ## Check the code
pyflakes *py pyflakes *py
black -t py37 -l 79 *py black -t py37 -l 79 *py
## SNI Support
https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484
\ No newline at end of file
...@@ -8,11 +8,11 @@ setup( ...@@ -8,11 +8,11 @@ setup(
include_package_data=False, include_package_data=False,
zip_safe=True, zip_safe=True,
install_requires=[ install_requires=[
"setuptools==40.5.0", "setuptools>40.5.0",
"requests==2.20.0", "requests>2.20.0",
"requests_toolbelt==0.8.0", "forcediphttpsadapter",
"peewee==2.10.1", "peewee>2.10.1",
"click==7.0", "click>=7.0",
], ],
entry_points={ entry_points={
"console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "] "console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]
......
from urllib.parse import urlparse from urllib.parse import urlparse, urlunsplit
import requests import requests
import socket import socket
import sys import sys
import traceback import traceback
import time import time
from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
__version__ = "0.0.1" __version__ = "0.0.2"
PREFERRED_TYPE = "text/html" PREFERRED_TYPE = "text/html"
TIMEOUT = 2 TIMEOUT = 2
...@@ -31,6 +32,8 @@ class WebBot: ...@@ -31,6 +32,8 @@ class WebBot:
stream=False, stream=False,
timeout=TIMEOUT, timeout=TIMEOUT,
allow_redirects=False, allow_redirects=False,
verify=True,
session=requests,
**kwargs, **kwargs,
): ):
...@@ -45,11 +48,13 @@ class WebBot: ...@@ -45,11 +48,13 @@ class WebBot:
kwargs["stream"] = stream kwargs["stream"] = stream
kwargs["timeout"] = timeout kwargs["timeout"] = timeout
kwargs["allow_redirects"] = allow_redirects kwargs["allow_redirects"] = allow_redirects
args = [url] kwargs["verify"] = verify
args = [method, url]
kwargs["headers"] = headers kwargs["headers"] = headers
try: try:
response = requests.get(*args, **kwargs) response = session.request(*args, **kwargs)
except requests.exceptions.SSLError: except requests.exceptions.SSLError:
# XXX Enter into unknown host # XXX Enter into unknown host
response = requests.models.Response() response = requests.models.Response()
...@@ -72,8 +77,9 @@ class WebBot: ...@@ -72,8 +77,9 @@ class WebBot:
return response return response
def check(self, url): def check(self, url):
print(url)
parsed_url = urlparse(url) parsed_url = urlparse(url)
response = self.request("GET", url)
print(url, response.status_code)
# Get the list of available IPv4 frontend CDN # Get the list of available IPv4 frontend CDN
hostname = parsed_url.hostname hostname = parsed_url.hostname
...@@ -81,12 +87,20 @@ class WebBot: ...@@ -81,12 +87,20 @@ class WebBot:
ip_list = [x[4][0] for x in dns_info_list] ip_list = [x[4][0] for x in dns_info_list]
for ip in ip_list: for ip in ip_list:
parsed_url = parsed_url._replace(netloc=ip) session = requests.Session()
ip_url = parsed_url.geturl() # SNI Support
if parsed_url.scheme == "https":
# Provide SNI support
base_url = urlunsplit(
(parsed_url.scheme, parsed_url.netloc, "", "", "")
)
session.mount(base_url, ForcedIPHTTPSAdapter(dest_ip=ip))
response = self.request("GET", ip_url, headers={"Host": hostname}) response = self.request(
"GET", url, headers={"Host": hostname}, session=session
)
print(ip, response.status_code) print(ip, hostname, response.status_code)
def run(self, url): def run(self, url):
print(time.strftime("%Y-%m-%d %H:%M:%S")) print(time.strftime("%Y-%m-%d %H:%M:%S"))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment