diff --git a/software/monitor/cgi-httpd.conf.in b/software/monitor/cgi-httpd.conf.in deleted file mode 100644 index 34fa34bc23d3f23b876bcf101a18178152b843b2..0000000000000000000000000000000000000000 --- a/software/monitor/cgi-httpd.conf.in +++ /dev/null @@ -1,112 +0,0 @@ -PidFile "{{ httpd_configuration.get('pid-file') }}" -ServerName example.com -ServerAdmin someone@email -<IfDefine !MonitorPort> -Listen [{{ httpd_configuration.get('listening-ip') }}]:{{ monitor_parameters.get('port') }} -Define MonitorPort -</IfDefine> -DocumentRoot "{{ directory.get('www') }}" -ErrorLog "{{ httpd_configuration.get('error-log') }}" -LoadModule unixd_module modules/mod_unixd.so -LoadModule access_compat_module modules/mod_access_compat.so -LoadModule authz_core_module modules/mod_authz_core.so -LoadModule authn_core_module modules/mod_authn_core.so -LoadModule authz_host_module modules/mod_authz_host.so -LoadModule mime_module modules/mod_mime.so -LoadModule cgid_module modules/mod_cgid.so -LoadModule dir_module modules/mod_dir.so -LoadModule ssl_module modules/mod_ssl.so -LoadModule alias_module modules/mod_alias.so -LoadModule autoindex_module modules/mod_autoindex.so -LoadModule auth_basic_module modules/mod_auth_basic.so -LoadModule authz_user_module modules/mod_authz_user.so -LoadModule authn_file_module modules/mod_authn_file.so -LoadModule proxy_module modules/mod_proxy.so -LoadModule proxy_http_module modules/mod_proxy_http.so -LoadModule rewrite_module modules/mod_rewrite.so -LoadModule setenvif_module modules/mod_setenvif.so -LoadModule headers_module modules/mod_headers.so -LoadModule env_module modules/mod_env.so -LoadModule socache_shmcb_module modules/mod_socache_shmcb.so - -SetEnvIf Origin "http(s)?://(www\.)?(.*)$" AccessControlAllowOrigin=$0 -Header always set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin -Header set Access-Control-Allow-Credentials "true" -Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST" -Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization" -Header set Access-Control-Expose-Headers "Content-Disposition, Content-Type, Content-Length, WWW-Authenticate, WWW-Logout, Location, Date, Last-Modified, ETag, Accept-Ranges, Content-Range" - - -# SSL Configuration -<IfDefine !SSLConfigured> -Define SSLConfigured -SSLCertificateFile {{ httpd_configuration.get('certificate') }} -SSLCertificateKeyFile {{ httpd_configuration.get('key') }} -SSLRandomSeed startup builtin -SSLRandomSeed connect builtin -SSLRandomSeed startup /dev/urandom 256 -SSLRandomSeed connect builtin -SSLProtocol all -SSLv2 -SSLv3 -SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5 -SSLHonorCipherOrder on -SSLSessionCache shmcb:/{{ directory.get("mod-ssl") }}/ssl_scache(512000) -SSLSessionCacheTimeout 300 -</IfDefine> -SSLEngine On -ScriptSock {{ httpd_configuration.get('cgid-pid-file') }} -<Directory {{ directory.get('www') }}> - SSLVerifyDepth 1 - SSLRequireSSL - SSLOptions +StrictRequire - # XXX: security???? - Options +ExecCGI - AddHandler cgi-script .cgi - DirectoryIndex {{ monitor_parameters.get('index-filename') }} -</Directory> - -Alias /{{ monitor_private_hash }} {{ directory.get('private-directory') }}/ -<Directory {{ directory.get('private-directory') }}> - Order Allow,Deny - Allow from all - AllowOverride All - Satisfy Any - Options Indexes FollowSymLinks - DirectoryIndex index.html - <Files .htaccess> - order allow,deny - deny from all - </Files> -</Directory> - -{% set slave_list = json_module.loads(slave_information.get('slave_instance_list', '{}')) -%} -{% for slave_instance in slave_list -%} - -alias /{{ slave_instance.get('slave_reference') }} {{ directory.get('private-directory') }}/network-user-logs/{{ slave_instance.get('slave_reference') }} -<Directory {{ directory.get('private-directory') }}/network-user-logs/{{ slave_instance.get('slave_reference') }}> - Order Allow,Deny - Allow from all - AllowOverride All - Satisfy Any - Options Indexes FollowSymLinks - DirectoryIndex index.html - <Files .htaccess> - order allow,deny - deny from all - </Files> -</Directory> - -{% endfor -%} - - -<Location /rewrite> -AuthType Basic -AuthName "Private access" -AuthUserFile "{{ monitor_parameters.get('htaccess-file') }}" -Require valid-user -</Location> - -ProxyVia On -RewriteEngine On -{% for key, value in monitor_rewrite_rule.iteritems() %} -RewriteRule ^/rewrite/{{ key }}($|/.*) {{ value }}/$1 [P,L] -{% endfor %} diff --git a/software/monitor/fluentd-agent.conf.jinja2.in b/software/monitor/fluentd-agent.conf.jinja2.in index f40cdadddba7c806133d6cfceb92725637350b1e..b4410e9b9027a85c016f32f17a18c90aefd042ea 100644 --- a/software/monitor/fluentd-agent.conf.jinja2.in +++ b/software/monitor/fluentd-agent.conf.jinja2.in @@ -13,104 +13,135 @@ </secondary> </match> -## match tag=debug.** and dump to console -<match debug.**> - type stdout +{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} +{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} +{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} +{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%} +{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} +<source> + type syslog + port 42185 + bind 127.0.0.1 + tag {{ wendelin_uid }}.system +</source> + +<match {{ wendelin_uid }}.system.**> + @type wendelin + @id wendelin_out + + streamtool_uri {{ wendelin_streamtool_uri }}_syslog + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s </match> <source> type tail - path {{ crawl_log_directory }}/*/*ping.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping.pos - tag slapos.monitor.networktest.ping.ipv4 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ + path {{ fluentd_log_directory }}/consumption/*/dump_user.csv + pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos + tag slapos.monitor.usage.user.{{ wendelin_uid }} + format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ read_from_head true </source> +<match slapos.monitor.usage.user.{{ wendelin_uid }}> + @type wendelin + @id wendelin_out + + streamtool_uri {{ wendelin_streamtool_uri }}_user + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s +</match> + <source> type tail - path {{ crawl_log_directory }}/*/*ping6.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.monitor.networktest.ping.ipv6 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ + path {{ fluentd_log_directory }}/consumption/*/dump_system.csv + pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos + tag slapos.monitor.usage.system.{{ wendelin_uid }} + format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ read_from_head true </source> -<match slapos.monitor.networktest.ping.*> - type copy -{% for slave_instance in slave_instance_list -%} -{% if slave_instance.get("ping_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% if slave_instance.get("ping6_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% endfor -%} -</match> +<match slapos.monitor.usage.system.{{ wendelin_uid }}> + @type wendelin + @id wendelin_out -{% for slave_instance in slave_instance_list -%} -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log - append true - include_time_key true - include_tag_key true - format json -</match> -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log - append true - include_time_key true - include_tag_key true - format json + streamtool_uri {{ wendelin_streamtool_uri }}_system + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s </match> -{% endfor -%} + <source> type tail - path {{ network_user_logs }}/*/ping*/log/*.log - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.wendelin.networktest - format json + path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv + pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos + tag slapos.monitor.usage.computer.{{ wendelin_uid }} + format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ + read_from_head true </source> -{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} -{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} -{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} -{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} -<match slapos.wendelin.networktest> - +<match slapos.monitor.usage.computer.{{ wendelin_uid }}> @type wendelin @id wendelin_out - streamtool_uri {{ wendelin_streamtool_uri }} + streamtool_uri {{ wendelin_streamtool_uri }}_computer user {{ wendelin_user }} password {{ wendelin_password }} buffer_type memory flush_interval 20s </match> -{% endif -%} -## SOURCE <source> - type forward + type tail + path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv + pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos + tag slapos.monitor.usage.disk.{{ wendelin_uid }} + format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ + read_from_head true </source> -## live debugging agent +<match slapos.monitor.usage.disk.{{ wendelin_uid }}> + @type wendelin + @id wendelin_out + + streamtool_uri {{ wendelin_streamtool_uri }}_disk + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s +</match> + <source> - type debug_agent - bind 127.0.0.1 - port 24230 + type tail + path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv + pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos + tag slapos.monitor.usage.temperature.{{ wendelin_uid }} + format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ + read_from_head true </source> +<match slapos.monitor.usage.temperature.{{ wendelin_uid }}> + @type wendelin + @id wendelin_out + + streamtool_uri {{ wendelin_streamtool_uri }}_temperature + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s +</match> + +{% endif -%} diff --git a/software/monitor/fluentd-distributor-agent.conf.jinja2.in b/software/monitor/fluentd-distributor-agent.conf.jinja2.in new file mode 100644 index 0000000000000000000000000000000000000000..f40cdadddba7c806133d6cfceb92725637350b1e --- /dev/null +++ b/software/monitor/fluentd-distributor-agent.conf.jinja2.in @@ -0,0 +1,116 @@ +# DESTINATION +<match td.*.*> + type tdlog + apikey YOUR_API_KEY + + auto_create_table + buffer_type file + buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td + + <secondary> + type file + path {{ fluentd_log_directory }}/failed_records + </secondary> +</match> + +## match tag=debug.** and dump to console +<match debug.**> + type stdout +</match> + +<source> + type tail + path {{ crawl_log_directory }}/*/*ping.log.20* + pos_file {{ crawl_log_directory }}/tail_in_ping.pos + tag slapos.monitor.networktest.ping.ipv4 + format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ + read_from_head true +</source> + +<source> + type tail + path {{ crawl_log_directory }}/*/*ping6.log.20* + pos_file {{ crawl_log_directory }}/tail_in_ping6.pos + tag slapos.monitor.networktest.ping.ipv6 + format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ + read_from_head true +</source> + +<match slapos.monitor.networktest.ping.*> + type copy +{% for slave_instance in slave_instance_list -%} +{% if slave_instance.get("ping_ip_list") -%} + <store> + type grep + regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$ + add_tag_prefix {{ slave_instance.get("slave_reference") }} + </store> +{% endif -%} +{% if slave_instance.get("ping6_ip_list") -%} + <store> + type grep + regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$ + add_tag_prefix {{ slave_instance.get("slave_reference") }} + </store> +{% endif -%} +{% endfor -%} +</match> + +{% for slave_instance in slave_instance_list -%} +<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6> + type file + path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log + append true + include_time_key true + include_tag_key true + format json +</match> +<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4> + type file + path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log + append true + include_time_key true + include_tag_key true + format json +</match> +{% endfor -%} + +<source> + type tail + path {{ network_user_logs }}/*/ping*/log/*.log + pos_file {{ crawl_log_directory }}/tail_in_ping6.pos + tag slapos.wendelin.networktest + format json + read_from_head true +</source> + +{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} +{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} +{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} +{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} +<match slapos.wendelin.networktest> + + @type wendelin + @id wendelin_out + + streamtool_uri {{ wendelin_streamtool_uri }} + user {{ wendelin_user }} + password {{ wendelin_password }} + + buffer_type memory + flush_interval 20s +</match> +{% endif -%} + +## SOURCE +<source> + type forward +</source> + +## live debugging agent +<source> + type debug_agent + bind 127.0.0.1 + port 24230 +</source> + diff --git a/software/monitor/index.cgi.in b/software/monitor/index.cgi.in deleted file mode 100755 index f560fb45d2c877a097d395141b7330725f27273a..0000000000000000000000000000000000000000 --- a/software/monitor/index.cgi.in +++ /dev/null @@ -1,6 +0,0 @@ -#!{{ extra_eggs_interpreter }} - -# Beginning of response -print "Content-Type: text/plain" -print -print "OK" diff --git a/software/monitor/instance-monitor-distributor.cfg.jinja2 b/software/monitor/instance-monitor-distributor.cfg.jinja2 index d178c21f24ab5738ca649a7e64418b4cf6e07c02..da04c34d21ecfe2c87a7911167db05de25bbf733 100644 --- a/software/monitor/instance-monitor-distributor.cfg.jinja2 +++ b/software/monitor/instance-monitor-distributor.cfg.jinja2 @@ -10,10 +10,10 @@ context = [slave-test-configuration] <=jinja2-template-base template = {{ template_json_distributor_test }} -filename = srv/monitor-private/test.json +filename = srv/monitor/private/test.json extensions = jinja2.ext.do extra-context = - section slave_information slap-parameter + section slave_information slap-configuration {% set part_list = [] -%} # Publish information for each slave @@ -27,7 +27,7 @@ extra-context = recipe = slapos.cookbook:publish -slave-reference = {{ slave_instance.get('slave_reference') }} log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }} -log-access-url-v6 = ${monitor-parameters:url}/{{ slave_instance.get('slave_reference') }} +log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }} {% endfor %} @@ -49,13 +49,12 @@ command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper {% endif %} [monitor-directory] -fluentd-log = ${:log}/fluentd crawl-log = ${:srv}/crawlog -network-user-logs = ${:private-directory}/network-user-logs/ +network-user-logs = ${:private}/network-user-logs/ {% for slave_instance in slave_instance_list -%} -user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }} -user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping -user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6 +user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }} +user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping +user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6 {% endfor -%} {% for directory in directory_list %} @@ -74,35 +73,26 @@ parameters-extra = true extends = {{ instance_base_monitor }} parts += slave-test-configuration - fluentd-wrapper + fluentd-distributor-wrapper {% for part in part_list %} {{ ' %s' % part }} {% endfor %} -[fluentd-wrapper] -recipe = slapos.cookbook:wrapper -command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered} -wrapper-path = ${monitor-directory:service}/fluentd -environment = - GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/ +[fluentd-distributor-wrapper] +<=fluentd-wrapper +command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered} +wrapper-path = ${monitor-directory:service}/fluentd-distributor -[fluentd-conf-configuration] +[fluentd-distributor-conf-configuration] recipe = slapos.recipe.template:jinja2 -template = {{ fluent_conf_output }} -rendered = ${monitor-directory:etc}/fluentd.cfg +template = {{ fluent_distributor_conf_output }} +rendered = ${monitor-directory:etc}/fluentd-distributor.cfg mode = 0744 context = key slapparameter_dict slap-configuration:configuration - key slave_instance_list slap-parameters:slave-instance-list + key slave_instance_list slap-configuration:slave-instance-list key fluentd_log_directory monitor-directory:fluentd-log key crawl_log_directory monitor-directory:crawl-log key network_user_logs monitor-directory:network-user-logs - -[slap-configuration] -recipe = slapos.cookbook:slapconfiguration.serialised -computer = ${slap-connection:computer-id} -partition = ${slap-connection:partition-id} -url = ${slap-connection:server-url} -key = ${slap-connection:key-file} -cert = ${slap-connection:cert-file} + key computer_id slap-configuration:computer diff --git a/software/monitor/instance-monitor.cfg.jinja2 b/software/monitor/instance-monitor.cfg.jinja2 index cac12df4050915c2645fde1106d47cc78590ab78..eb27814a13a58386f343a805930c43be561d7a26 100644 --- a/software/monitor/instance-monitor.cfg.jinja2 +++ b/software/monitor/instance-monitor.cfg.jinja2 @@ -1,28 +1,16 @@ [buildout] +extends = {{ monitor_template_output }} + parts = - publish-connection-informations cron - certificate-authority - cron-entry-monitor - cron-entry-rss - deploy-index - deploy-settings-cgi - deploy-status-cgi - deploy-status-history-cgi - setup-static-files - certificate-authority - zero-parameters - public-symlink - cgi-httpd-wrapper - cgi-httpd-graceful-wrapper - monitor-promise - monitor-instance-log-access cron-network-bench symlink-re6st-logs symlink-collected-logs - -extends = {{ monitor_template_output }} + monitor-collect-csv-wrapper + fluentd-wrapper + monitor-base + publish-connection-information eggs-directory = {{ eggs_directory }} @@ -52,8 +40,7 @@ template = {{ network_benck_cfg_output }} rendered = ${monitor-directory:etc}/network_bench.cfg mode = 0744 context = - key slapparameter_dict slap-parameters:configuration - + key slapparameter_dict slap-configuration:configuration [pwgen] recipe = slapos.cookbook:generate.password @@ -66,38 +53,16 @@ user = admin bytes = 16 [monitor-directory] -server-log = ${:private-directory}/server-log -monitor-log = ${:private-directory}/monitor-log +service = ${buildout:directory}/etc/service +var = ${buildout:directory}/var +srv = ${buildout:directory}/srv +server-log = ${:private}/server-log +monitor-log = ${:private}/monitor-log cache = ${:var}/cache mod-ssl = ${:cache}/httpd_mod_ssl -system-log = ${:private-directory}/system-log - -[slap-parameter] -private-hash = ${pwgen:passwd}${pwgen32:passwd} -frontend-domain = - -[monitor-frontend] -recipe = slapos.cookbook:requestoptional -name = Monitor Frontend -# XXX We have hardcoded SR URL here. -software-url = product.frontend -software-type = custom-personal -slave = true -config-url = ${monitor-parameters:url} -config-domain = ${slap-parameter:frontend-domain} -return = site_url domain - -server-url = ${slap-connection:server-url} -key-file = ${slap-connection:key-file} -cert-file = ${slap-connection:cert-file} -computer-id = ${slap-connection:computer-id} -partition-id = ${slap-connection:partition-id} - -[publish-connection-informations] -recipe = slapos.cookbook:publish -monitor_url = ${monitor-parameters:url} -server_log_url = ${:url}${slap-parameter:private-hash}/ -url = ${monitor-frontend:connection-site_url} +system-log = ${:private}/system-log +fluentd-log = ${:log}/fluentd +consumption = ${:fluentd-log}/consumption [monitor-httpd-configuration-file] context = @@ -106,6 +71,47 @@ context = section monitor_parameters monitor-parameters section httpd_configuration monitor-httpd-configuration section monitor_rewrite_rule monitor-rewrite-rule - section slave_information slap-parameter - key monitor_private_hash slap-parameter:private-hash + section slave_information slap-configuration + key monitor_private_hash slap-configuration:private-hash +[fluentd-wrapper] +recipe = slapos.cookbook:wrapper +command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered} +wrapper-path = ${monitor-directory:service}/fluentd +environment = + GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/ + +[fluentd-conf-configuration] +recipe = slapos.recipe.template:jinja2 +template = {{ fluent_conf_output }} +rendered = ${monitor-directory:etc}/fluentd.cfg +mode = 0744 +context = + key slapparameter_dict slap-configuration:configuration + key slave_instance_list slap-configuration:slave-instance-list + key fluentd_log_directory monitor-directory:fluentd-log + key computer_id slap-connection:computer-id + + +[monitor-collect-csv-wrapper] +recipe = slapos.cookbook:wrapper +command-line = + ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/ +wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump +parameters-extra = true + +[publish-connection-information] +recipe = slapos.cookbook:publish +monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password} +server_log_url = ${publish:monitor-base-url}/${slap-configuration:private-hash}/ + + +[slap-configuration] +recipe = slapos.cookbook:slapconfiguration.serialised +computer = ${slap-connection:computer-id} +partition = ${slap-connection:partition-id} +url = ${slap-connection:server-url} +key = ${slap-connection:key-file} +cert = ${slap-connection:cert-file} +private-hash = ${pwgen:passwd}${pwgen32:passwd} +frontend-domain = diff --git a/software/monitor/instance.cfg b/software/monitor/instance.cfg index b38f2699528affbbfa65fbdc939d26ee93b2c07d..d6ade910a1ee8a29f110207387b5953ff206531b 100644 --- a/software/monitor/instance.cfg +++ b/software/monitor/instance.cfg @@ -13,7 +13,7 @@ distributor = $${instance-base-distributor:rendered} [instance-base-monitor] recipe = slapos.recipe.template:jinja2 template = ${template-monitor:destination} -rendered = $${buildout:directory}/template-monitor.cfg +rendered = $${buildout:directory}/template-base-monitor.cfg extensions = jinja2.ext.do context = key buildout buildout:bin-directory key develop_eggs_directory buildout:develop-eggs-directory @@ -21,12 +21,15 @@ context = key buildout buildout:bin-directory key slapparameter_dict slap-configuration:configuration raw monitor_template_output ${monitor-template:output} raw network_benck_cfg_output ${network-bench-cfg:output} + raw fluentd_location ${fluentd:location} + raw fluent_conf_output ${fluentd-agent-conf:output} + raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} mode = 0644 [instance-base-distributor] recipe = slapos.recipe.template:jinja2 template = ${template-monitor-distributor:destination} -rendered = $${buildout:directory}/template-monitor-distributor.cfg +rendered = $${buildout:directory}/template-monitor-base-distributor.cfg extensions = jinja2.ext.do context = import json_module json key buildout buildout:bin-directory @@ -35,9 +38,9 @@ context = import json_module json key slapparameter_dict slap-configuration:configuration key instance_base_monitor instance-base-monitor:rendered key slave_instance_list slap-configuration:slave-instance-list - raw template_json_distributor_test ${json-test-template:destination} - raw fluent_conf_output ${fluentd-agent-conf:output} + raw template_json_distributor_test ${json-test-template:destination} raw fluentd_location ${fluentd:location} + raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output} raw wget_bin ${wget:location}/bin/wget mode = 0644 @@ -48,3 +51,4 @@ partition = $${slap-connection:partition-id} url = $${slap-connection:server-url} key = $${slap-connection:key-file} cert = $${slap-connection:cert-file} + diff --git a/software/monitor/script/collect_csv_dump.py b/software/monitor/script/collect_csv_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..085176a9b19ed726cef75b45537ad156ea2d1ad6 --- /dev/null +++ b/software/monitor/script/collect_csv_dump.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (c) 2010-2016 Vifib SARL and Contributors. +# All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation; either version 2.1 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import os +import argparse +import csv + +from slapos.util import mkdir_p +from slapos.collect.db import Database + +def skip_bootstrap(self): + return + +Database._bootstrap = skip_bootstrap + +def parseArguments(): + """ + Parse arguments for monitor collector instance. + """ + parser = argparse.ArgumentParser() + parser.add_argument('--output_folder', + help='Path of the folder where output files should be written.') + parser.add_argument('--collector_db', + default='/srv/slapgrid/var/data-log/', + help='The path of slapos collect database is located.') + + return parser.parse_args() + +def writeFile(name, folder, date_scope, rows): + if os.path.exists( + os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))): + # File already exists, no reason to recreate it. + return + mkdir_p(os.path.join(folder, date_scope), 0o755) + file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w") + csv_output = csv.writer(file_io) + csv_output.writerows(rows) + file_io.close() + +def dump_table_into_csv(db, folder): + db.connect() + table_list = db.getTableList() + # Save all dates first, as db.selector may switch the cursor + date_list = [(date_scope, _) \ + for date_scope, _ in db.getDateScopeList(reported=1)] + + for date_scope, amount in date_list: + for table in table_list: + if os.path.exists( + os.path.join(folder, "%s/dump_%s.csv" % (date_scope, table))): + # File already exists, no reason to recreate it. + continue + writeFile(table, folder, date_scope, + db.select(table, date_scope)) + db.close() + + +if __name__ == "__main__": + parser = parseArguments() + if parser.output_folder is None: + raise Exception("Invalid ouput folder: %s" % parser.output_folder) + + if parser.collector_db is None: + raise Exception("Invalid collector database folder: %s" % parser.collector_db) + + + if not os.path.exists(parser.output_folder) and \ + os.path.isdir(parser.output_folder): + raise Exception("Invalid ouput folder: %s" % parser.output_folder) + + + if not os.path.exists(parser.collector_db): + print "Collector database not found..." + + dump_table_into_csv(Database(parser.collector_db), parser.output_folder) diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index bab8805bd4ce115d9695ada06981af06588a1a6a..48e50b3b479ff8816c8dbf4d78b71702ca4ed1ed 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -11,35 +11,35 @@ extends = parts = wget slapos-cookbook - slapos-toolbox network-bench-cfg json-test-template template template-monitor-distributor template-monitor - eggs fluentd - fluentd-agent-conf + fluentd-agent-conf + fluentd-agent-distributor-conf + monitor-collect-csv-dump [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg output = ${buildout:directory}/template.cfg -md5sum = b38c5da7dd4165154a8817f76015e374 +md5sum = 0c5e94beede86a91d7b01f61a7290f86 mode = 0644 [template-monitor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 -destination = ${buildout:directory}/template-monitor.cfg -md5sum = 810ed8199682068e27b62659d7fa101f +destination = ${buildout:directory}/template-base-monitor.cfg +md5sum = 4649ce8bc496cf7560c2e994f0b332ee mode = 0644 [template-monitor-distributor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 -destination = ${buildout:directory}/template-monitor-distributor.cfg -md5sum = d1253cf11c2b335b8bec5807e821d537 +destination = ${buildout:directory}/template-monitor-base-distributor.cfg +md5sum = f579fe67fa1670b46d522b1e5f35c49c mode = 0644 [json-test-template] @@ -49,16 +49,6 @@ destination = ${buildout:directory}/json-test-template.json.in.jinja2 md5sum = 2eb5596544d9c341acf653d4f7ce2680 mode = 0644 - -# stupify index for now -[index] -url = ${:_profile_base_location_}/${:filename} -md5sum = 876f18b159fbd9325332d0f42e9172ac - -[monitor-httpd-template] -url = ${:_profile_base_location_}/${:filename} -md5sum = e89b66a90409bb8e9099aa197803d337 - [network-bench-cfg] recipe = slapos.recipe.template url = ${:_profile_base_location_}/network_bench.cfg.in @@ -69,17 +59,33 @@ mode = 0644 [fluentd-agent-conf] recipe = slapos.recipe.template url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in -md5sum = 4b81ddcbe2f16d23013caac37151d396 +md5sum = 3ea59906937eab7aeef78f46c4994ecd output = ${buildout:directory}/fluentd-agent.conf.jinja2.in mode = 0644 -[slapos-toolbox] -recipe = zc.recipe.egg -eggs = +[fluentd-agent-distributor-conf] +recipe = slapos.recipe.template +url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in +md5sum = 4b81ddcbe2f16d23013caac37151d396 +output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in +mode = 0644 + +[monitor-collect-csv-dump] +<= monitor-template-script +url = ${:_profile_base_location_}/script/${:filename} +filename = collect_csv_dump.py +output = ${:destination}/${:filename} +md5sum = cad2402bbd21907cfed6bc5af8c5d3ab + +[extra-eggs] +<= monitor-eggs +interpreter = pythonwitheggs +eggs += ${lxml-python:egg} ${pycurl:egg} ${python-cryptography:egg} slapos.toolbox + slapos.core scripts = networkbench @@ -89,43 +95,11 @@ scripts = gems += fluent-plugin-wendelin==0.1.alpha1 fluent-plugin-grep==0.3.4 + fluent-plugin-bin==0.1 -[eggs] -recipe = zc.recipe.egg -eggs = - cns.recipe.symlink +[monitor-eggs] +eggs += slapos.cookbook slapos.recipe.cmmi hexagonit.recipe.download plone.recipe.command - -[versions] -PyRSS2Gen = 1.1 -apache-libcloud = 1.0.0rc2 -cns.recipe.symlink = 0.2.3 -gitdb = 0.6.4 -pycurl = 7.43.0 -rubygemsrecipe = 0.2.2 -slapos.recipe.template = 2.9 -slapos.toolbox = 0.55 -smmap = 0.9.0 - -# Required by: -# slapos.toolbox==0.55 -GitPython = 2.0.5 - -# Required by: -# slapos.toolbox==0.55 -atomize = 0.2.0 - -# Required by: -# slapos.toolbox==0.55 -feedparser = 5.2.1 - -# Required by: -# slapos.toolbox==0.55 -lockfile = 0.12.2 - -# Required by: -# slapos.toolbox==0.55 -paramiko = 2.0.0