apache-frontend: replace squid by apache traffic server for caching

apache-frontend: ats: add basic cache configuration

XXX RAM (1G) and disk (8G) cache size Should not be hardcoded

apache-frontend: ats: mark in header status of request in ats

apache-frontend: remove squid

software/apache-frontend/common.cfg

@@ -13,7 +13,8 @@ extends =
   ../../component/dcron/buildout.cfg
   ../../component/logrotate/buildout.cfg
   ../../component/rdiff-backup/buildout.cfg
-  ../../component/squid/buildout.cfg
+  ../../component/trafficserver/buildout.cfg
+
 # Monitoring stack
   ../../stack/monitor/buildout.cfg
 
@@ -33,7 +34,6 @@ parts +=
   dcron
   logrotate
   rdiff-backup
-  squid
 
 [slapos-toolbox]
 recipe = zc.recipe.egg
@@ -67,14 +67,14 @@ mode = 0644
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = f5ec3d3b29d20ccdb00e3b64aa588fa5
+#md5sum = f5ec3d3b29d20ccdb00e3b64aa588fa5
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = 82c88a4b4856bfffec3d7ef24e372f38
+#md5sum = 82c88a4b4856bfffec3d7ef24e372f38
 mode = 0644
 
 [template-slave-list]
@@ -144,12 +144,6 @@ url = ${:_profile_base_location_}/templates/template-log-access.conf.in
 md5sum = f85005b430978f3bd24ee7ce11b0e304
 mode = 640
 
-[template-squid-configuration]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/templates/squid.conf.jinja2
-md5sum = f17753fa87da074bc949b2967a330099
-mode = 640
-
 [template-empty]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/empty.in
@@ -162,3 +156,12 @@ url = ${:_profile_base_location_}/templates/wrapper.in
 output = ${buildout:directory}/template-wrapper.cfg
 mode = 0644
 md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
+
+[template-trafficserver-records-config]
+recipe = hexagonit.recipe.download
+url = ${:_profile_base_location_}/templates/trafficserver/${:filename}
+#md5sum = 788795524769f6d946526ac282508b69
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+filename = records.config.jinja2
+download-only = true
+mode = 0644

software/apache-frontend/instance-apache-frontend.cfg

@@ -9,16 +9,11 @@ parts =
   certificate-authority
   logrotate-entry-apache
   logrotate-entry-apache-cached
-  logrotate-entry-squid
   apache-frontend
   apache-cached
   switch-apache-softwaretype
   frontend-apache-graceful
   cached-apache-graceful
-  squid-service
-  squid-prepare
-  squid-reload
-  promise-squid
   dynamic-template-default-vh
   not-found-html
   promise-frontend-apache-configuration
@@ -28,6 +23,14 @@ parts =
   promise-apache-frontend-v6-https
   promise-apache-frontend-v6-http
   promise-apache-cached
+
+  trafficserver-launcher
+  trafficserver-reload
+  trafficserver-configuration-directory
+  trafficserver-records-config
+  trafficserver-remap-config
+  trafficserver-storage-config
+
 ## Monitoring part
 ###Parts to add for monitoring
   certificate-authority
@@ -79,6 +82,7 @@ crontabs = $${:etc}/crontabs
 cronstamps = $${:etc}/cronstamps
 ca-dir = $${:srv}/ssl
 
+
 [switch-apache-softwaretype]
 recipe = slapos.cookbook:softwaretype
 single-default = $${dynamic-default-template-slave-list:rendered}
@@ -135,6 +139,7 @@ context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key slap_software_type instance-parameter:slap-software-type
     key slapparameter_dict instance-parameter:configuration
+    section directory directory
     $${:extra-context}
 
 [dynamic-template-default-vh]
@@ -333,8 +338,8 @@ cache-access-log = $${directory:log}/frontend-apache-access-cached.log
 cache-error-log = $${directory:log}/frontend-apache-error-cached.log
 cache-pid-file = $${directory:run}/httpd-cached.pid
 
-# Comunication with squid
-cache-port = 26010
+# Comunication with ats
+cache-port = $${trafficserver-variable:input-port}
 cache-through-port = 26011
 
 # Create wrapper for "apachectl conftest" in bin
@@ -433,77 +438,68 @@ sharedscripts = true
 notifempty = true
 create = true
 
-[logrotate-entry-squid]
-<= logrotate
-recipe = slapos.cookbook:logrotate.d
-name = squid
-log = $${squid-cache:cache-log-path} $${squid-cache:access-log-path}
-frequency = daily
-rotatep-num = 30
-post = ${buildout:bin-directory}/killpidfromfile $${apache-configuration:pid-file} SIGHUP
-sharedscripts = true
-notifempty = true
-create = true
-
-######################
-#  Squid deployment
-######################
-[squid-directory]
+#################
+# Trafficserver
+#################
+[trafficserver-directory]
 recipe = slapos.cookbook:mkdirectory
-squid-cache = $${directory:srv}/squid_cache
-
-[squid-cache]
-prepare-path = $${directory:etc-run}/squid-prepare
-wrapper-path = $${directory:service}/squid
-binary-path = ${squid:location}/sbin/squid
-configuration-path = $${directory:etc}/squid.cfg
-cache-path = $${squid-directory:squid-cache}
-ip = $${instance-parameter:ipv4-random}
-port = $${apache-configuration:cache-port}
-backend-ip = $${instance-parameter:ipv4-random}
-backend-port = $${apache-configuration:cache-through-port}
-open-port = $${instance-parameter:configuration.open-port}
-access-log-path = $${directory:log}/squid-access.log
-cache-log-path = $${directory:log}/squid-cache.log
-pid-filename-path = $${directory:run}/squid.pid
-
-[squid-configuration]
-< = jinja2-template-base
-template = ${template-squid-configuration:target}
-rendered = $${squid-cache:configuration-path}
-extra-context =
-      key ip squid-cache:ip
-      key port squid-cache:port
-      key backend_ip squid-cache:backend-ip
-      key backend_port squid-cache:backend-port
-      key cache_path squid-cache:cache-path
-      key access_log_path squid-cache:access-log-path
-      key cache_log_path squid-cache:cache-log-path
-      key pid_filename_path squid-cache:pid-filename-path
-      key open_port squid-cache:open-port
-
-[squid-service]
+configuration = $${directory:etc}/trafficserver
+local-state = $${directory:var}/trafficserver
+bin_path = ${trafficserver:location}/bin
+log = $${directory:log}/trafficserver
+cache-path = $${directory:srv}/ats_cache
+
+[trafficserver-variable]
+wrapper-path = $${directory:service}/trafficserver
+reload-path = $${directory:etc-run}/trafficserver-reload
+local-ip = $${instance-parameter:ipv4-random}
+input-port = 23432
+hostname = $${slap-parameter:frontend-name}
+remap = map / http://$${instance-parameter:ipv4-random}:$${apache-configuration:cache-through-port}
+disk-cache-config = $${trafficserver-directory:cache-path} 8G volume=$${slap-parameter:frontend-name}
+
+[trafficserver-configuration-directory]
+recipe = plone.recipe.command
+command = cp -rn ${trafficserver:location}/etc/trafficserver/* $${:target}
+target = $${trafficserver-directory:configuration}
+
+[trafficserver-launcher]
 recipe = slapos.cookbook:wrapper
-command-line = $${squid-cache:binary-path} -N -f $${squid-configuration:rendered}
-wrapper-path = $${squid-cache:wrapper-path}
+command-line = ${trafficserver:location}/bin/traffic_cop
+wrapper-path = $${trafficserver-variable:wrapper-path}
+environment = TS_ROOT=$${buildout:directory}
 
-[squid-prepare]
+[trafficserver-reload]
 recipe = slapos.cookbook:wrapper
-command-line = $${squid-cache:binary-path} -z -f $${squid-configuration:rendered}
-wrapper-path = $${squid-cache:prepare-path}
+command-line = ${trafficserver:location}/bin/traffic_line -x
+wrapper-path = $${trafficserver-variable:reload-path}
+environment = TS_ROOT=$${buildout:directory}
 
-[squid-reload]
-recipe = slapos.cookbook:wrapper
-command-line = ${buildout:bin-directory}/killpidfromfile $${squid-cache:pid-filename-path} SIGHUP
-wrapper-path = $${directory:etc-run}/squid-reload
+[trafficserver-records-config]
+< = jinja2-template-base
+template = ${template-trafficserver-records-config:location}/${template-trafficserver-records-config:filename}
+rendered = $${trafficserver-directory:configuration}/records.config
+mode = 700
+context =
+    import os_module os
+    section ats_directory trafficserver-directory
+    section ats_configuration trafficserver-variable
 
-[promise-squid]
-recipe = slapos.cookbook:check_port_listening
-path = $${directory:promise}/squid
-hostname = $${instance-parameter:ipv4-random}
-port = $${apache-configuration:cache-port}
+[trafficserver-remap-config]
+< = jinja2-template-base
+template = ${template-empty:target}
+rendered = $${trafficserver-configuration-directory:target}/remap.config
+mode = 700
+context =
+    key content trafficserver-variable:remap
 
-# End of Squid part
+[trafficserver-storage-config]
+< = jinja2-template-base
+template = ${template-empty:target}
+rendered = $${trafficserver-configuration-directory:target}/storage.config
+mode = 700
+context =
+    key content trafficserver-variable:disk-cache-config
 
 ### Apaches Graceful and promises
 [frontend-apache-graceful]
@@ -577,3 +573,21 @@ server_url = $${slap-connection:server-url}
 software_release_url = $${slap-connection:software-release-url}
 key_file = $${slap-connection:key-file}
 cert_file = $${slap-connection:cert-file}
+
+[slap-parameter]
+# Define default parameter(s) that will be used later, in case user didn't
+# specify it
+# All parameters are available through the configuration.XX syntax.
+# All possible parameters should have a default.
+domain = example.org
+public-ipv4 =
+port = 4443
+plain_http_port = 8080
+server-admin = admin@example.com
+apache_custom_https = ""
+apache_custom_http = ""
+apache-key =
+apache-certificate =
+open-port = 80 443
+extra_slave_instance_list =
+frontend-name =
\ No newline at end of file

software/apache-frontend/templates/squid.conf.jinja2

-refresh_pattern .   0 20% 4320 max-stale=604800
-
-# Dissallow cachemgr access
-http_access deny manager
-
-# Squid service configuration
-http_port {{ ip }}:{{ port }} accel defaultsite={{ ip }}
-
-cache_peer {{ backend_ip }} parent {{ backend_port }} 0 no-query originserver name=backend
-
-acl our_sites port {{ open_port }}
-http_access allow our_sites
-cache_peer_access backend allow our_sites
-cache_peer_access backend deny all
-
-# Drop squid headers
-# via off
-# reply_header_access X-Cache-Lookup deny all
-# reply_header_access X-Squid-Error deny all
-# reply_header_access X-Cache deny all
-
-header_replace X-Forwarded-For
-follow_x_forwarded_for allow all
-forwarded_for on
-
-cache_dir aufs {{ cache_path }} 5000 16 256
-
-# Use 1Go of RAM
-cache_mem 1024 MB
-# But do not keep big object in RAM
-maximum_object_size_in_memory 2048 KB
-
-# Log
-access_log {{ access_log_path }}
-cache_log {{ cache_log_path }}
-pid_filename {{ pid_filename_path }}