From f2b48450c8a881184ba1eaf43c9c5854e376080b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Le=20Ninivin?= <cedric.leninivin@tiolive.com> Date: Fri, 26 Apr 2013 10:33:28 +0200 Subject: [PATCH] squid: Add Squid to Apache Frontend (Romain) --- slapos/recipe/apache_frontend/__init__.py | 147 ++++++++++++++++-- .../apache_frontend/template/apache.conf.in | 12 +- .../apache_frontend/template/squid.conf.in | 34 ++++ software/apache-frontend/common.cfg | 4 +- software/apache-frontend/instance.cfg | 1 + 5 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 slapos/recipe/apache_frontend/template/squid.conf.in diff --git a/slapos/recipe/apache_frontend/__init__.py b/slapos/recipe/apache_frontend/__init__.py index 47641fd78..c196e6fa7 100644 --- a/slapos/recipe/apache_frontend/__init__.py +++ b/slapos/recipe/apache_frontend/__init__.py @@ -129,15 +129,17 @@ class Recipe(BaseSlapRecipe): slave_dict[reference] = "%s%s/" % (scheme, domain) # Check if we want varnish+stunnel cache. - #if enable_cache: - # # XXX-Cedric : need to refactor to clean code? (to many variables) - # rewrite_rule = self.configureVarnishSlave( - # base_varnish_port, backend_url, reference, service_dict, domain) - # base_varnish_port += 2 - #else: - # rewrite_rule = "%s %s" % (domain, backend_url) - # Temporary forbid activation of cache until it is properly tested - rewrite_rule = "%s %s" % (domain, backend_url) + if enable_cache: + # XXX-Cedric : need to refactor to clean code? (to many variables) +# rewrite_rule = self.configureVarnishSlave( +# base_varnish_port, backend_url, reference, service_dict, domain) + rewrite_rule = self.configureSquidSlave( + base_varnish_port, backend_url, reference, service_dict, domain) + base_varnish_port += 2 + else: + rewrite_rule = "%s %s" % (domain, backend_url) +# # Temporary forbid activation of cache until it is properly tested +# rewrite_rule = "%s %s" % (domain, backend_url) # Finally, if successful, we add the rewrite rule to our list of rules # We have 4 RewriteMaps: @@ -250,6 +252,133 @@ class Recipe(BaseSlapRecipe): return self.path_list + def configureSquidSlave(self, base_squid_port, url, reference, + service_dict, domain): + # Squid should use stunnel to connect to the backend + base_squid_control_port = base_squid_port + base_squid_port += 1 + # Use regex + host_regex = "((\[\w*|[0-9]+\.)(\:|)).*(\]|\.[0-9]+)" + slave_host = re.search(host_regex, url).group(0) + port_regex = "\w+(\/|)$" + matcher = re.search(port_regex, url) + if matcher is not None: + slave_port = matcher.group(0) + slave_port = slave_port.replace("/", "") + elif url.startswith("https://"): + slave_port = 443 + else: + slave_port = 80 + service_name = "squid_%s" % reference + squid_ip = self.getLocalIPv4Address() + stunnel_port = base_squid_port + 1 + self.installSquidCache(service_name, + ip=squid_ip, + port=base_squid_port, + backend_host=squid_ip, + backend_port=stunnel_port, + domain=domain, + size="1G") + service_dict[service_name] = dict(public_ip=squid_ip, + public_port=stunnel_port, + private_ip=slave_host.replace("[", "").replace("]", ""), + private_port=slave_port) + return "%s http://%s:%s" % \ + (domain, squid_ip, base_squid_port) + + def installSquidCache(self, name, ip, port, backend_host, + backend_port, domain, size="1G"): + """ + Install a squid daemon for a certain address + """ +# directory = self.createDataDirectory(name) +# squid_config = dict( +# directory=directory, +# pid = "%s/squid.pid" % directory, +# port="%s:%s" % (ip, port), +# squidd_binary=self.options["squidd_binary"], +# control_port="%s:%s" % (ip, control_port), +# storage="file,%s/storage.bin,%s" % (directory, size)) + +# +# squid_argument_list = [squid_config['squidd_binary'].strip(), +# "-F", "-n", directory, "-P", squid_config["pid"], "-p", +# "cc_command=exec %s " % self.options["gcc_binary"] +\ +# "-fpic -shared -o %o %s", +# "-f", config_file, +# "-a", squid_config["port"], "-T", squid_config["control_port"], +# "-s", squid_config["storage"]] +# environment = dict(PATH="%s:%s" % (self.options["binutils_directory"], +# os.environ.get('PATH'))) +# wrapper = zc.buildout.easy_install.scripts([(name, +# 'slapos.recipe.librecipe.execute', 'executee')], self.ws, +# sys.executable, self.service_directory, arguments=[squid_argument_list, +# environment])[0] +# self.path_list.append(wrapper) + + +# directory = self.createDataDirectory(name) + config = dict( + ip=ip, + port=port, + backend_ip=backend_host, + backend_port=backend_port, + domain=domain, + # XXX Hardcoded + access_log_path = os.path.join(self.log_directory, 'squid.access.log'), + # XXX Hardcoded + cache_log_path = os.path.join(self.log_directory, 'squid.cache.log'), +# cache_path=self.options['cache-path'], + # XXX Hardcoded + pid_filename_path=os.path.join(self.run_directory, 'squid.pid'), + squid_binary=self.options["squid_binary"], + ) + + template_filename = self.getTemplateFilename('squid.conf.in') + config_file = self.createConfigurationFile("%s.conf" % name, + self.substituteTemplate(self.getTemplateFilename('squid.conf.in'), + config)) + +# # Prepare directories +# prepare_path = self.createPythonScript( +# self.options['prepare-path'], +# 'slapos.recipe.librecipe.execute.execute', +# arguments=[self.options['binary-path'].strip(), +# '-z', +# '-f', configuration_path, +# ],) +# +# # Create running wrapper +# wrapper_path = self.createPythonScript( +# self.options['wrapper-path'], +# 'slapos.recipe.librecipe.execute.execute', +# arguments=[self.options['binary-path'].strip(), +# '-N', +# '-f', configuration_path, +# ],) +# +# return [configuration_path, wrapper_path, prepare_path] + + squid_argument_list = [config['squid_binary'].strip(), + "-N", "-f", config_file] +# "cc_command=exec %s " % self.options["gcc_binary"] +\ +# "-fpic -shared -o %o %s", +# "-f", config_file, +# "-a", config["port"], "-T", config["control_port"], +# "-s", config["storage"]] + environment = dict(PATH="%s:%s" % (self.options["binutils_directory"], + os.environ.get('PATH'))) + wrapper = zc.buildout.easy_install.scripts([(name, + 'slapos.recipe.librecipe.execute', 'executee')], self.ws, + sys.executable, self.service_directory, arguments=[squid_argument_list, + environment])[0] + self.path_list.append(wrapper) + + return config + + + + def configureVarnishSlave(self, base_varnish_port, url, reference, service_dict, domain): # Varnish should use stunnel to connect to the backend diff --git a/slapos/recipe/apache_frontend/template/apache.conf.in b/slapos/recipe/apache_frontend/template/apache.conf.in index 69c5e334f..cc71644af 100644 --- a/slapos/recipe/apache_frontend/template/apache.conf.in +++ b/slapos/recipe/apache_frontend/template/apache.conf.in @@ -22,10 +22,12 @@ ServerTokens Prod # Log configuration ErrorLog "%(error_log)s" -LogLevel info -LogFormat "%%h %%{REMOTE_USER}i %%{Host}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined -LogFormat "%%h %%{REMOTE_USER}i %%{Host}i %%l %%u %%t \"%%r\" %%>s %%b" common -CustomLog "%(access_log)s" common +LogLevel warn +# LogFormat "%%h %%{REMOTE_USER}i %%{Host}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined +# LogFormat "%%h %%{REMOTE_USER}i %%{Host}i %%l %%u %%t \"%%r\" %%>s %%b" common +# CustomLog "%(access_log)s" common +LogFormat "%%h %%l %%{REMOTE_USER}i %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined +CustomLog "%(access_log)s" combined %(path_enable)s @@ -149,7 +151,7 @@ Header append Vary User-Agent # Include configuration file not operated by slapos. This file won't be erased # or changed when slapgrid is ran. It can be freely customized by node admin. - Include %(custom_apache_virtualhost_conf)s + # Include %(custom_apache_virtualhost_conf)s RewriteMap apachemapgeneric txt:%(apachemap_path)s RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >"" diff --git a/slapos/recipe/apache_frontend/template/squid.conf.in b/slapos/recipe/apache_frontend/template/squid.conf.in new file mode 100644 index 000000000..ce49fb21b --- /dev/null +++ b/slapos/recipe/apache_frontend/template/squid.conf.in @@ -0,0 +1,34 @@ +refresh_pattern . 0 20%% 4320 max-stale=604800 + +# Dissallow cachemgr access +http_access deny manager + +# Squid service configuration +http_port %(ip)s:%(port)s accel defaultsite=%(ip)s + +cache_peer %(backend_ip)s parent %(backend_port)s 0 no-query originserver name=backend + +acl our_sites dstdomain %(domain)s +http_access allow our_sites +cache_peer_access backend allow our_sites +cache_peer_access backend deny all + +# Drop squid headers +# via off +# reply_header_access X-Cache-Lookup deny all +# reply_header_access X-Squid-Error deny all +# reply_header_access X-Cache deny all + +header_replace X-Forwarded-For +follow_x_forwarded_for allow all +forwarded_for on + +# Use 1Go of RAM +cache_mem 1024 MB +# But do not keep big object in RAM +maximum_object_size_in_memory 2048 KB + +# Log +access_log %(access_log_path)s +cache_log %(cache_log_path)s +pid_filename %(pid_filename_path)s diff --git a/software/apache-frontend/common.cfg b/software/apache-frontend/common.cfg index 4bd2736f4..49e6d2e3b 100644 --- a/software/apache-frontend/common.cfg +++ b/software/apache-frontend/common.cfg @@ -9,6 +9,7 @@ extends = ../../component/dcron/buildout.cfg ../../component/logrotate/buildout.cfg ../../component/rdiff-backup/buildout.cfg + ../../component/squid/buildout.cfg ../../stack/slapos.cfg parts = @@ -23,6 +24,7 @@ parts = dcron logrotate rdiff-backup + squid # Buildoutish eggs @@ -48,6 +50,6 @@ eggs = # Default template for apache instance. recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg -md5sum = e7b9f57da7eb1450fc15789e239388d4 +md5sum = 07e51c4be2c298db3bca151605698130 output = ${buildout:directory}/template.cfg mode = 0644 diff --git a/software/apache-frontend/instance.cfg b/software/apache-frontend/instance.cfg index 1f788b445..07f337e73 100644 --- a/software/apache-frontend/instance.cfg +++ b/software/apache-frontend/instance.cfg @@ -36,6 +36,7 @@ logrotate_binary = ${logrotate:location}/usr/sbin/logrotate openssl_binary = ${openssl:location}/bin/openssl dcrond_binary = ${dcron:location}/sbin/crond varnishd_binary = ${varnish-2.1:location}/sbin/varnishd +squid_binary = ${squid:location}/sbin/squid stunnel_binary = ${stunnel:location}/bin/stunnel rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup gcc_binary = gcc -- 2.30.9