From 5227882cd742ab964dc829099bca76dbc36f13c6 Mon Sep 17 00:00:00 2001 From: Tatuya Kamada <tatuya@nexedi.com> Date: Tue, 21 Feb 2012 17:00:29 +0900 Subject: [PATCH] Add varnish recipe. --- setup.py | 1 + slapos/recipe/generic_varnish/__init__.py | 77 +++++++++++ .../generic_varnish/template/default.vcl.in | 124 ++++++++++++++++++ .../generic_varnish/template/varnishd.in | 15 +++ .../generic_varnish/template/varnishlog.in | 8 ++ .../generic_varnish/template/varnishlogd.in | 22 ++++ software/erp5/instance-erp5-production.cfg | 9 ++ software/erp5/instance-varnish.cfg | 54 ++++++++ software/erp5/instance.cfg | 1 + software/erp5/software.cfg | 11 +- 10 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 slapos/recipe/generic_varnish/__init__.py create mode 100644 slapos/recipe/generic_varnish/template/default.vcl.in create mode 100644 slapos/recipe/generic_varnish/template/varnishd.in create mode 100644 slapos/recipe/generic_varnish/template/varnishlog.in create mode 100644 slapos/recipe/generic_varnish/template/varnishlogd.in create mode 100644 software/erp5/instance-varnish.cfg diff --git a/setup.py b/setup.py index d73cf83915..e71906684a 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ setup(name=name, 'tidstorage = slapos.recipe.tidstorage:Recipe', 'erp5.update = slapos.recipe.erp5_update:Recipe', 'erp5.test = slapos.recipe.erp5_test:Recipe', + 'generic.varnish = slapos.recipe.generic_varnish:Recipe', ], 'slapos.recipe.nosqltestbed.plugin': [ 'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed', diff --git a/slapos/recipe/generic_varnish/__init__.py b/slapos/recipe/generic_varnish/__init__.py new file mode 100644 index 0000000000..4187b7ea09 --- /dev/null +++ b/slapos/recipe/generic_varnish/__init__.py @@ -0,0 +1,77 @@ +############################################################################## +# +# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import os +import re +from slapos.recipe.librecipe import GenericSlapRecipe + +class Recipe(GenericSlapRecipe): + """ + Instantiate varnish daemon + + TODO: + - use varnish3.x and replace .vcl for it + """ + def _install(self): + ip = self.options['ip'] + backend_url = self.parameter_dict['tidstorage-url'] + backend_ip, backend_port = self._getBackendServer(backend_url) + varnishd_manager_port = int(self.options['manager-port']) + varnishd_server_port = int(self.options['server-port']) + path_list = [] + config = dict( + varnishd_binary=self.options['varnishd-binary'], + varnish_ip=ip, + varnishlog_binary=self.options['varnishlog-binary'], + varnishd_manager_port=varnishd_manager_port, + varnishd_server_port=varnishd_server_port, + varnishd_pid_file=self.options['pid-file'], + varnish_instance_name=self.options['varnish-instance-name'], + varnish_data=self.options['varnish-data'], + shell_path=self.options['shell-path'], + vcl_file=self.options['vcl-file'], + backend_ip = backend_ip, + backend_port = backend_port, + backend_server = "[%s]" % backend_ip, + ) + + path_list.append(self.createExecutable(self.options['varnishd-wrapper'], + self.substituteTemplate(self.getTemplateFilename('varnishd.in'), + config))) + path_list.append(self.createExecutable(self.options['varnishlog-wrapper'], + self.substituteTemplate(self.getTemplateFilename('varnishlog.in'), + config))) + path_list.append(self.createFile(self.options['vcl-file'], + self.substituteTemplate(self.getTemplateFilename('default.vcl.in'), + config))) + return path_list + + def _getBackendServer(self, url): + r = re.compile('\/\/\[(.*)\]:(\d*)') + result = r.search(url) + ip = result.groups()[0] + port = result.groups()[1] + return (ip, port) diff --git a/slapos/recipe/generic_varnish/template/default.vcl.in b/slapos/recipe/generic_varnish/template/default.vcl.in new file mode 100644 index 0000000000..bac86d2dc5 --- /dev/null +++ b/slapos/recipe/generic_varnish/template/default.vcl.in @@ -0,0 +1,124 @@ +#This is a basic VCL configuration file for varnish. See the vcl(7) +#man page for details on VCL syntax and semantics. +# +#Default backend definition. Set this to point to your content +#server. +# +backend default { + .host = "%(backend_ip)s"; + .port = "%(backend_port)s"; + .probe = { + .timeout = 30s; + .interval = 5s; + .window = 4; + .threshold = 3; + .request = + "OPTIONS /erp5/getId HTTP/1.1" + "Host: %(backend_server)s:%(backend_port)s" + "Accept-Encoding: identity" + "Connection: close" + "User-Agent: Varnish"; + } +} +# +#Below is a commented-out copy of the default VCL logic. If you +#redefine any of these subroutines, the built-in logic will be +#appended to your code. +# + +sub vcl_recv { + if (req.request != "GET" && + req.request != "HEAD" && + req.request != "PUT" && + req.request != "POST" && + req.request != "TRACE" && + req.request != "OPTIONS" && + req.request != "PURGE" && + req.request != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + pipe; + } + if (req.request != "GET" && req.request != "HEAD" && req.request != "PURGE") { + /* We only deal with GET and HEAD by default */ + pass; + } + remove req.http.Cookie; + remove req.http.Set-Cookie; + if (req.http.Accept-Encoding) { + if (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } else { + # unkown algorithm + remove req.http.Accept-Encoding; + } + } + # Force deflate + remove req.http.Accept-Encoding; + # We do not care about Accept-Language, this is url controlled + remove req.http.Accept-Language; + #if (req.request == "PURGE") { + # if (!client.ip ~ purge) { + # error 405 "Not allowed."; + # } + # purge_url(req.url); + # error 200 "HASHPURGED"; + # unset req.http.x; + #} + set req.grace = 30d; + lookup; +} + +sub vcl_hash { + set req.hash += req.url; + hash; +} + +sub vcl_hit { + #if (req.request == "PURGE" && client.ip ~ purge) { + # set obj.ttl = 0s; + # error 200 "Purged."; + #} + + #if (client.ip ~ purge){ + # # Force refresh from localhost + # set obj.ttl = 0s; + # return (restart); + #} + # According Vary Header do not return those headers + remove req.http.Accept-Language; + remove req.http.Accept-Encoding; + remove req.http.Cookie; + deliver; +} + +sub vcl_miss { + fetch; +} + +sub vcl_fetch { + /* Never send request to backend even if client ask refreshed content */ + if (obj.cacheable) { + /* Setup grace period for 30days for all cacheable contents */ + #set req.grace = 30d; + set obj.grace = 30d; + } + deliver; + } + + +sub vcl_deliver { + if (obj.hits > 0) { + set resp.http.X-Cache = obj.hits; + } else { + set resp.http.X-Cache = "MISS"; + } + #if (obj.hash) { + # set resp.http.X-Hash = obj.hash; + #} else { + # set resp.http.X-Hash = "No hash"; + #} + + deliver; +} diff --git a/slapos/recipe/generic_varnish/template/varnishd.in b/slapos/recipe/generic_varnish/template/varnishd.in new file mode 100644 index 0000000000..c30582488b --- /dev/null +++ b/slapos/recipe/generic_varnish/template/varnishd.in @@ -0,0 +1,15 @@ +#!%(shell_path)s + +DAEMON_OPTS="-F \ + -a %(varnish_ip)s:%(varnishd_server_port)s \ + -T %(varnish_ip)s:%(varnishd_manager_port)s \ + -n %(varnish_instance_name)s \ + -f %(vcl_file)s \ + -s file,%(varnish_data)s/varnish_storage.bin,1G" + +PIDFILE=%(varnishd_pid_file)s +# exporting PATH here so that we will pass the PATH variable to the subprocess +export PATH +output=$(/bin/tempfile -s.varnish) +exec %(varnishd_binary)s -P ${PIDFILE} ${DAEMON_OPTS} > ${output} 2>&1 + diff --git a/slapos/recipe/generic_varnish/template/varnishlog.in b/slapos/recipe/generic_varnish/template/varnishlog.in new file mode 100644 index 0000000000..c5a5f3d03e --- /dev/null +++ b/slapos/recipe/generic_varnish/template/varnishlog.in @@ -0,0 +1,8 @@ +#!%(shell_path)s + +DAEMON_OPTS="-a %(varnish_ip)s:%(varnishd_server_port)s \ + -n %(varnish_instance_name)s" + +# exporting PATH here so that it will pass the PATH variable to the subprocess +export PATH +%(varnishlog_binary)s ${DAEMON_OPTS} diff --git a/slapos/recipe/generic_varnish/template/varnishlogd.in b/slapos/recipe/generic_varnish/template/varnishlogd.in new file mode 100644 index 0000000000..b0f1e24dca --- /dev/null +++ b/slapos/recipe/generic_varnish/template/varnishlogd.in @@ -0,0 +1,22 @@ +#!%(shell_path)s + +DAEMON_OPTS="-F \ + -a %(varnish_ip)s:%(varnishd_server_port)s \ + -T %(varnish_ip)s:%(varnishd_manager_port)s \ + -n %(varnish_instance_name)s \ + -f %(vcl_file)s \ + -s file,%(varnish_data)s/varnish_storage.bin,1G" + +PIDFILE=%(varnishd_pid_file)s +# exporting PATH here so that it will pass the PATH variable to the subprocess +export PATH + +# If unset, or set to "0" or "no", exit +if [ -z "${VARNISHLOG_ENABLED}" ] || \ + [ "${VARNISHLOG_ENABLED}" = "0" ] || \ + [ "${VARNISHLOG_ENABLED}" = "no" ]; then + exit 0; +fi + +output=$(/bin/tempfile -s.varnish) +exec %(varnishlog_binary)s ${DAEMON_OPTS} > ${output} 2>&1 diff --git a/software/erp5/instance-erp5-production.cfg b/software/erp5/instance-erp5-production.cfg index cbf2451676..cb4c660096 100644 --- a/software/erp5/instance-erp5-production.cfg +++ b/software/erp5/instance-erp5-production.cfg @@ -74,6 +74,14 @@ config-kumofs-url = $${request-kumofs:connection-url} software-type = tidstorage sla-computer_guid = $${slap-parameter:tidstorage-computer-guid} +[request-varnish] +<=request-common +name = Varnish +config = tidstorage-url +config-tidstorage-url = $${request-tidstorage:connection-url-login} +software-type = varnish +sla-computer_guid = $${slap-parameter:varnish-computer-guid} + [slap-parameter] # Default value if no computer_guid is specified for each type sphinx-computer-guid = $${slap-connection:computer-id} @@ -82,6 +90,7 @@ cloudooo-computer-guid = $${slap-connection:computer-id} memcached-computer-guid = $${slap-connection:computer-id} kumofs-computer-guid = $${slap-connection:computer-id} tidstorage-computer-guid = $${slap-connection:computer-id} +varnish-computer-guid = $${slap-connection:computer-id} cloudooo-json = # rest of parts are candidates for some generic stuff diff --git a/software/erp5/instance-varnish.cfg b/software/erp5/instance-varnish.cfg new file mode 100644 index 0000000000..646a6da212 --- /dev/null +++ b/software/erp5/instance-varnish.cfg @@ -0,0 +1,54 @@ +[buildout] +parts = + publish-varnish-connection-information + varnish-instance + +eggs-directory = ${buildout:eggs-directory} +develop-eggs-directory = ${buildout:develop-eggs-directory} +offline = true + +[publish-varnish-connection-information] +recipe = slapos.cookbook:publishurl +url = http://$${varnish-instance:ip}:$${varnish-instance:server-port}/ + +[varnish-instance] +recipe = slapos.cookbook:generic.varnish + +# Network options +ip = $${slap-network-information:local-ipv4} +server-port = 6001 +manager-port = 6002 + +# Paths: Running wrappers +varnishd-wrapper = $${basedirectory:services}/varnishd +varnishlog-wrapper = $${rootdirectory:bin}/varnishlog + +# Binary information +varnishd-binary = ${varnish:location}/sbin/varnishd +varnishlog-binary = ${varnish:location}/bin/varnishlog +shell-path = ${dash:location}/bin/dash +# Configuration by VCL +vcl-file = $${rootdirectory:etc}/default.vcl +pid-file = $${basedirectory:run}/varnishd.pid +varnish-data = $${directory:varnish-data} +# this will pass at -n option +varnish-instance-name = $${directory:varnish-instance} + +[basedirectory] +recipe = slapos.cookbook:mkdirectory +services = $${rootdirectory:etc}/run +run = $${rootdirectory:var}/run +backup = $${rootdirectory:srv}/backup + +[directory] +recipe = slapos.cookbook:mkdirectory +varnish-data = $${rootdirectory:srv}/varnish +varnish-instance = $${directory:varnish-data}/instance + +[rootdirectory] +recipe = slapos.cookbook:mkdirectory +etc = $${buildout:directory}/etc +var = $${buildout:directory}/var +srv = $${buildout:directory}/srv +bin = $${buildout:directory}/bin + diff --git a/software/erp5/instance.cfg b/software/erp5/instance.cfg index 8abb090d57..82d61d9e31 100644 --- a/software/erp5/instance.cfg +++ b/software/erp5/instance.cfg @@ -33,6 +33,7 @@ zope = ${template-zope:output} mariadb = ${template-mariadb:output} sphinx = ${template-sphinx:output} tidstorage = $${dynamic-template-tidstorage:output} +varnish = ${template-varnish:output} [slap-connection] # part to migrate to new - separated words diff --git a/software/erp5/software.cfg b/software/erp5/software.cfg index a7483de7c7..6575f5ed9e 100644 --- a/software/erp5/software.cfg +++ b/software/erp5/software.cfg @@ -87,7 +87,7 @@ mode = 0644 [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg -md5sum = 961d404f5726fce7c4d8b34d7e120077 +md5sum = f3443d071d8d7330b3453583e096fef0 output = ${buildout:directory}/template.cfg mode = 0644 @@ -108,7 +108,7 @@ mode = 0644 [template-erp5-production] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-erp5-production.cfg -md5sum = ae697a5c82f80388c674217d5e6f61a0 +md5sum = 6bf9242195b6fe4e5fb5d080cefef255 output = ${buildout:directory}/template-erp5-production.cfg mode = 0644 @@ -140,6 +140,13 @@ md5sum = 396053ffe2904ad9933a42fe2f8cfcde output = ${buildout:directory}/template-snippet-backend.cfg mode = 0644 +[template-varnish] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/instance-varnish.cfg +md5sum = f708845296a1e9ea1c7fa699ace67a30 +output = ${buildout:directory}/template-varnish.cfg +mode = 0644 + [validator] # Default json schema for instance parameters. recipe = slapos.recipe.template -- 2.30.9