Commit 340a7609 authored by Jérome Perrin's avatar Jérome Perrin

software/*: version up urllib3, requests and dependencies

Several softwares were using fixed versions, but versions are also defined in
stack/slapos.cfg, so let's use from stack/slapos.cfg which will simplify
version management.

This urllib3 has visible changes for us, it no longer send the request path
"as is", but normalizes the . and .. which is not what we want in
caddy-frontend test, so we adjust the test to use prepared requests, which is
the recommended approach for this kind of low level needs. Since prepared
requests does not set default user agent "python-requests", tests checking that
user agent is in log files have been adjusted to use a dedicated test user agent
header.

Another change is that certificates are now verified by default, so we adjust
seleniumserver tests to disable certificate validation. jstestnode, already
had a patch regarding certificate validation, because we wanted certificates
to be validated before it was the default. We just update this patch a bit
because we want to keep the optional behaviours of validating against a
specific CA or to disable validation.
parent 0f5d6bd7
......@@ -228,15 +228,6 @@ mode = 0644
[versions]
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
# Versions pinned for kedifa need urllib3 >= 1.18
urllib3 = 1.24
requests = 2.20.0
certifi = 2018.10.15
idna = 2.7
chardet = 3.0.4
# ipaddress is patching IPAddress so IPv6 match works
ipaddress = 1.0.22
# Versions pinned for kedifa need urllib3 >= 1.18
validators = 0.12.2
PyRSS2Gen = 1.1
......
......@@ -419,13 +419,21 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
try:
add_custom_dns(domain, port, TEST_IP)
socket.getaddrinfo = new_getaddrinfo
return session.get(
'https://%s:%s/%s' % (domain, port, path),
verify=False,
allow_redirects=False,
# Use a prepared request, to disable path normalization.
# We need this because some test checks requests with paths like
# /test-path/deep/.././deeper but we don't want the client to send
# /test-path/deeper
# See also https://github.com/psf/requests/issues/5289
url = 'https://%s:%s/%s' % (domain, port, path)
req = requests.Request(
method='GET',
url=url,
headers=headers,
cookies=cookies
cookies=cookies,
)
prepped = req.prepare()
prepped.url = url
return session.send(prepped, verify=False, allow_redirects=False)
finally:
socket.getaddrinfo = socket_getaddrinfo
......@@ -447,11 +455,13 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
new_source = source.SourceAddressAdapter(source_ip)
session.mount('http://', new_source)
session.mount('https://', new_source)
return session.get(
'http://%s:%s/%s' % (TEST_IP, port, path),
headers=headers,
allow_redirects=False,
)
# Use a prepared request, to disable path normalization.
url = 'http://%s:%s/%s' % (TEST_IP, port, path)
req = requests.Request(method='GET', url=url, headers=headers)
prepped = req.prepare()
prepped.url = url
return session.send(prepped, allow_redirects=False)
class TestHandler(BaseHTTPRequestHandler):
......@@ -1821,6 +1831,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
headers={
'Timeout': '10', # more than default backend-connect-timeout == 5
'Accept-Encoding': 'gzip',
'User-Agent': 'TEST USER AGENT',
}
)
......@@ -1854,7 +1865,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
log_regexp = r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} - - ' \
r'\[\d{2}\/.{3}\/\d{4}\:\d{2}\:\d{2}\:\d{2} \+\d{4}\] ' \
r'"GET \/test-path\/deep\/..\/.\/deeper HTTP\/1.1" \d{3} ' \
r'\d+ "-" "python-requests.*" \d+'
r'\d+ "-" "TEST USER AGENT" \d+'
self.assertRegexpMatches(
open(log_file, 'r').readlines()[-1],
......
......@@ -750,7 +750,7 @@ class TestFrontendXForwardedFor(BalancerTestCase):
).json()
self.assertNotEqual(result['Incoming Headers'].get('x-forwarded-for', '').split(', ')[0], '1.2.3.4')
balancer_url = json.loads(self.computer_partition.getConnectionParameterDict()['_'])['default-auth']
with self.assertRaises(OpenSSL.SSL.Error):
with self.assertRaisesRegexp(Exception, "certificate required"):
requests.get(
balancer_url,
headers={'X-Forwarded-For': '1.2.3.4'},
......
......@@ -27,4 +27,4 @@ md5sum = 98faa5ad8cfb23a11d97a459078a1d05
[template-runTestSuite]
filename = runTestSuite.in
md5sum = bb3f053b6cdb0a8888e9d32e63085ed5
md5sum = 73ef758825563b7a6a1d660d4d5229b8
......@@ -97,8 +97,8 @@ def main():
executable_path='${geckodriver:location}')
else:
assert target == 'selenium-server', "Unsupported target {}".format(test_runner['target'])
# use a remote connection which verifies TLS certificate
# workaround for https://github.com/SeleniumHQ/selenium/issues/6534
# use a remote connection which optionally verifies TLS certificate
# and optionally against a user provided CA.
executor = RemoteConnection(test_runner['server-url'], keep_alive=True)
cert_reqs = 'CERT_REQUIRED'
ca_certs = certifi.where()
......
......@@ -117,7 +117,4 @@ output = ${buildout:directory}/template-nginx.cfg.in
output = ${buildout:directory}/runTestSuite.in
[versions]
selenium = 3.14.1
urllib3 = 1.24
certifi = 2018.10.15
......@@ -81,12 +81,7 @@ surykatka = 0.5.0
# For surykatka 0.5.0
click = 7.0
certifi = 2019.11.28
chardet = 3.0.4
dnspython = 1.16.0
forcediphttpsadapter = 1.0.1
idna = 2.8
miniupnpc = 2.0.2
peewee = 3.13.1
requests = 2.22.0
urllib3 = 1.25.8
......@@ -46,8 +46,10 @@ from PIL import Image
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.remote.remote_connection import RemoteConnection
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import urllib3
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort, ImageComparisonTestCase, ManagedHTTPServer
......@@ -318,8 +320,13 @@ class TestFrontend(WebServerMixin, SeleniumServerTestCase):
self.assertEqual('selenium', parsed.username)
self.assertTrue(parsed.password)
# XXX we are using a self signed certificate, but selenium 3.141.0 does
# not expose API to ignore certificate verification
executor = RemoteConnection(webdriver_url, keep_alive=True)
executor._conn = urllib3.PoolManager(cert_reqs='CERT_NONE', ca_certs=None)
driver = webdriver.Remote(
command_executor=webdriver_url,
command_executor=executor,
desired_capabilities=DesiredCapabilities.CHROME)
driver.get(self.server_url)
......
......@@ -298,16 +298,7 @@ selenium = 3.141.0
# Patched eggs
PyPDF2 = 1.26.0+SlapOSPatched001
# modern versions for SSL fixed support
requests = 2.20.0
certifi = 2018.10.15
idna = 2.7
chardet = 3.0.4
# ipaddress is patching IPAddress so IPv6 match works
ipaddress = 1.0.22
# cacuase and its dependencies
# caucase and its dependencies
caucase = 0.9.4
pem = 18.2.0
PyJWT = 1.6.4
......@@ -315,10 +306,6 @@ PyJWT = 1.6.4
# Django 1.11 is python 2 compatible
Django = 1.11
# Required by:
# selenium==3.141.0
urllib3 = 1.24.1
mock = 2.0.0
testfixtures = 6.11
funcsigs = 1.0.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment