instance-monitor-distributor.cfg.jinja2 3.73 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:filename}
extra-context =
context =
    import json_module json
    ${:extra-context}

[slave-test-configuration]
<=jinja2-template-base
template = {{ template_json_distributor_test }}
filename = srv/monitor-private/test.json
extensions = jinja2.ext.do 
extra-context = 
  section slave_information slap-parameter

{% set part_list = [] -%}
# Publish information for each slave

21 22
{% set directory_list = [] -%}

23 24 25 26 27 28 29 30 31 32
{% for slave_instance in slave_instance_list -%}
{%   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
{%   do part_list.append(publish_section_title) -%}
[{{ publish_section_title }}]
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }}
log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
log-access-url-v6 = ${monitor-parameters:url}/{{ slave_instance.get('slave_reference') }}
{% endfor %}

33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72

{% set data_source_dict = slapparameter_dict.get('data-source', None)  -%}
{% set cron_min_count = 0 -%}
{% if data_source_dict %}
{% for entry in data_source_dict -%}
{% set cron_min_count = cron_min_count + 1 -%}
{% set cron_min = cron_min_count%60 -%}
{% do part_list.append('cron-crawl-' + entry) -%}
{% do directory_list.append(entry) -%}
[cron-crawl-{{ entry }}]
<= cron
recipe = slapos.cookbook:cron.d
name = cron-crawler-{{ entry }}
frequency = * * * * *
command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }} 
{% endfor %}
{% endif %}

[monitor-directory]
fluentd-log = ${:log}/fluentd
crawl-log = ${:srv}/crawlog
network-user-logs = ${:private-directory}/network-user-logs/
{% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}
user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6

{% endfor -%}
{% for directory in directory_list %}
{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
{% endfor %}


[crawler-bin]
recipe = slapos.cookbook:wrapper
command-line = 
  {{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp  
wrapper-path = ${monitor-directory:bin}/log-crawler
parameters-extra = true

73
[buildout]
74
extends = {{ instance_base_monitor }}
75
parts +=
76
  slave-test-configuration
77
  fluentd-wrapper
78 79 80
{% for part in part_list %}
{{ '  %s' % part }}
{% endfor %}
81 82 83 84 85 86 87 88 89 90 91 92 93 94

[fluentd-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd
environment = 
  GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/

[fluentd-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
95 96
  key slapparameter_dict slap-configuration:configuration
  key slave_instance_list slap-parameters:slave-instance-list
97
  key fluentd_log_directory monitor-directory:fluentd-log
98 99 100 101 102 103 104 105 106 107 108
  key crawl_log_directory monitor-directory:crawl-log
  key network_user_logs monitor-directory:network-user-logs

[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}