diff --git a/component/caddy/buildout.cfg b/component/caddy/buildout.cfg index 7b91f78952601a2f2020f1fb7304503e01419472..c0838a966cc0bc6ba4b3294051607a0867967a9c 100644 --- a/component/caddy/buildout.cfg +++ b/component/caddy/buildout.cfg @@ -8,6 +8,8 @@ parts = [gowork] # Caddy 1.x+ uses go modules, for which gowork does not work yet golang = ${golang1.12:location} +gcc-bin-directory = ${golang1.12:gcc-bin-directory} + install = [gowork.goinstall] diff --git a/component/golang/buildout.cfg b/component/golang/buildout.cfg index dbe457f5f9bf808a78324b96a883508be9e3fcd2..6c97125175d9bf722addd4afc1c3bdff00ec25c8 100644 --- a/component/golang/buildout.cfg +++ b/component/golang/buildout.cfg @@ -17,10 +17,12 @@ configure-command = : location = @@LOCATION@@ make-binary = make-targets= cd src && ./all.bash && cp -alf .. ${:location} +# gcc version to use +gcc-bin-directory = ${gcc-8.2:location}/bin # some testdata files have an issue with slapos.extension.strip. post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true environment = - PATH=${gcc-8.2:location}/bin:%(PATH)s + PATH=${:gcc-bin-directory}:%(PATH)s GOROOT_FINAL=${:location} ${:environment-extra} @@ -104,6 +106,9 @@ depends = ${gowork.goinstall:recipe} # go version used for the workspace (possible to override in applications) golang = ${golang1.10:location} +# gcc version must be compatible with go version selected +gcc-bin-directory = ${golang1.10:gcc-bin-directory} + # no special build flags by default buildflags = @@ -123,7 +128,7 @@ recipe = slapos.recipe.template url = ${:_profile_base_location_}/goenv.sh.in output = ${gowork:directory}/env.sh depends = ${gowork.mkdir:recipe} -md5sum = 7a067a3974c446c3eaa0e82818ba1adb +md5sum = 7eaad1f9aabd3cfad554975098c5d4c3 [gowork.mkdir] # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance) diff --git a/component/golang/goenv.sh.in b/component/golang/goenv.sh.in index 4c0e98b4b8ed0ed1670ae55a87165a225b8e2e09..89e43825b252d912d5a0a2ab5e245c9ceb71caf4 100644 --- a/component/golang/goenv.sh.in +++ b/component/golang/goenv.sh.in @@ -3,7 +3,7 @@ # ---- 8< ---- (buildout substitution here) # PATH so that go & friends work out of the box -export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH +export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:${gowork:gcc-bin-directory}:$PATH X=${gowork:directory} export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH diff --git a/component/helloweb/buildout.cfg b/component/helloweb/buildout.cfg index 7710c466cbd11aca4244bf22d0e24b71cbc93832..43a79c62e0adf6a90530c5be00a4fbc12be17d9e 100644 --- a/component/helloweb/buildout.cfg +++ b/component/helloweb/buildout.cfg @@ -26,7 +26,7 @@ install = lab.nexedi.com/nexedi/helloweb/go/... golang = ${golang1.12:location} - +gcc-bin-directory = ${golang1.12:gcc-bin-directory} # -*- go -*- [helloweb-go] diff --git a/software/grafana/README.md b/software/grafana/README.md index 1c6e8ff59dfe09925dd201b5ec2ae9d4d227834d..b125897efdd60d786b60a6ed22fa8f540d56bdfb 100644 --- a/software/grafana/README.md +++ b/software/grafana/README.md @@ -1,30 +1,27 @@ # grafana / telegraf / influxdb - +This is an experimental integration, mainly to evaluate these solutions. + ## Custom telegraf plugins See https://github.com/influxdata/telegraf to learn about plugins. Useful plugins in this context are probably -[exec](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/exec) +[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec), +[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser) or -[httpjson](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/httpjson). +[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http). Telegraf will save in the `telegraf` database from the embedded influxdb server. ## Grafana -You'll have to add yourself the influxdb data source in grafana, using the -parameters published by the slapos instance. - -http://docs.grafana.org/features/datasources/influxdb/ +A default user is created, username and password are published as connection +parameters. You can add more users in grafana interface. -When adding datasource, use *proxy* option, otherwise Grafana makes your -browser query influxdb directly, which also uses a self signed certificate. -One workaround is to configure your browser to also accept influxdb certificate -before using grafana, but using proxy seems easier. +Datasources should be automatically added. ## Influxdb @@ -34,8 +31,32 @@ One important thing to notice is that the backup protocol is enabled on ipv4 provided by slapos, so make sure this ip is not reachable from untrusted sources. -## TODO +# Ingesting/Visualizing logs + +Eventhough main feature is visualizing metrics, Grafana has a feature called "Explore" to view logs for a time frame. +The following backend can be used: + +## Loki + +See `TestLoki` in test for an example. + +## Influxdb + +Influxdb logs only have tags and there does not seem to be a way to search (except than tag and time frame). + +To inject log files containing: +``` +INFO the message +WARN another message +``` + +use config like: + +``` +[[inputs.logparser]] + files = ["/tmp/x*.log", "/tmp/aaa.log"] -* influxdb and telegraf runs with very low priority, this could become an option -* make one partition for each service and use switch software type -* make it easier to add custom configuration (how ?) + [inputs.logparser.grok] + measurement = "logs" + patterns = ['^%{WORD:level:tag} %{GREEDYDATA:message:string}'] +``` diff --git a/software/grafana/buildout.hash.cfg b/software/grafana/buildout.hash.cfg index 01f73b27c1594925213382d3c3cce931d74e4003..a7761213bf50ea6de85c6e90a607dae09c4fe5cc 100644 --- a/software/grafana/buildout.hash.cfg +++ b/software/grafana/buildout.hash.cfg @@ -15,7 +15,7 @@ [instance-profile] filename = instance.cfg.in -md5sum = 461d515da03de5e422e6f75189d09184 +md5sum = 214a756a6a9f73455411760b2daa25b7 [influxdb-config-file] filename = influxdb-config-file.cfg.in @@ -28,3 +28,15 @@ md5sum = a1a9c22c2a7829c66a49fc2504604d21 [grafana-config-file] filename = grafana-config-file.cfg.in md5sum = 8244d430905b968795c7946049bed9e3 + +[grafana-provisioning-config-file] +filename = grafana-provisioning-config-file.cfg.in +md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3 + +[loki-config-file] +filename = loki-config-file.cfg.in +md5sum = 02ba5acf23fcf88f5594919f46838533 + +[promtail-config-file] +filename = promtail-config-file.cfg.in +md5sum = c77788d0a3cc654ad9393eb4b1f31e94 diff --git a/software/grafana/gowork.cfg b/software/grafana/gowork.cfg index a93d605e7cf974711650803b449e60382969a26c..84d6544c2364372cd6017dd755fba879bf1068d5 100644 --- a/software/grafana/gowork.cfg +++ b/software/grafana/gowork.cfg @@ -3,844 +3,39 @@ # list of go git repositories to fetch [gowork.goinstall] depends_gitfetch = - ${go_collectd.org:recipe} - ${go_github.com_BurntSushi_toml:recipe} - ${go_github.com_Microsoft_go-winio:recipe} - ${go_github.com_RoaringBitmap_roaring:recipe} - ${go_github.com_Shopify_sarama:recipe} - ${go_github.com_Sirupsen_logrus:recipe} - ${go_github.com_StackExchange_wmi:recipe} - ${go_github.com_aerospike_aerospike-client-go:recipe} - ${go_github.com_amir_raidman:recipe} - ${go_github.com_apache_thrift:recipe} - ${go_github.com_aws_aws-sdk-go:recipe} - ${go_github.com_beorn7_perks:recipe} - ${go_github.com_bmizerany_pat:recipe} - ${go_github.com_boltdb_bolt:recipe} - ${go_github.com_bsm_sarama-cluster:recipe} - ${go_github.com_cenkalti_backoff:recipe} - ${go_github.com_cespare_xxhash:recipe} - ${go_github.com_couchbase_go-couchbase:recipe} - ${go_github.com_couchbase_gomemcached:recipe} - ${go_github.com_couchbase_goutils:recipe} - ${go_github.com_davecgh_go-spew:recipe} - ${go_github.com_dgrijalva_jwt-go:recipe} - ${go_github.com_dgryski_go-bitstream:recipe} - ${go_github.com_docker_docker:recipe} - ${go_github.com_docker_go-connections:recipe} - ${go_github.com_eapache_go-resiliency:recipe} - ${go_github.com_eapache_go-xerial-snappy:recipe} - ${go_github.com_eapache_queue:recipe} - ${go_github.com_eclipse_paho.mqtt.golang:recipe} - ${go_github.com_glycerine_go-unsnap-stream:recipe} - ${go_github.com_go-ini_ini:recipe} - ${go_github.com_go-logfmt_logfmt:recipe} - ${go_github.com_go-ole_go-ole:recipe} - ${go_github.com_go-sql-driver_mysql:recipe} - ${go_github.com_gobwas_glob:recipe} - ${go_github.com_gogo_protobuf:recipe} - ${go_github.com_golang_protobuf:recipe} - ${go_github.com_golang_snappy:recipe} - ${go_github.com_google_go-cmp:recipe} - ${go_github.com_gorilla_mux:recipe} + ${go_github.com_golang_dep:recipe} ${go_github.com_grafana_grafana:recipe} - ${go_github.com_hailocab_go-hostpool:recipe} - ${go_github.com_hashicorp_consul:recipe} + ${go_github.com_grafana_loki:recipe} ${go_github.com_influxdata_influxdb:recipe} - ${go_github.com_influxdata_influxql:recipe} - ${go_github.com_influxdata_tail:recipe} ${go_github.com_influxdata_telegraf:recipe} - ${go_github.com_influxdata_toml:recipe} - ${go_github.com_influxdata_usage-client:recipe} - ${go_github.com_influxdata_wlog:recipe} - ${go_github.com_influxdata_yamux:recipe} - ${go_github.com_influxdata_yarpc:recipe} - ${go_github.com_jackc_pgx:recipe} - ${go_github.com_jmespath_go-jmespath:recipe} - ${go_github.com_jwilder_encoding:recipe} - ${go_github.com_kardianos_govendor:recipe} - ${go_github.com_kardianos_osext:recipe} - ${go_github.com_kardianos_service:recipe} - ${go_github.com_kballard_go-shellquote:recipe} - ${go_github.com_matttproud_golang_protobuf_extensions:recipe} - ${go_github.com_miekg_dns:recipe} - ${go_github.com_mitchellh_mapstructure:recipe} - ${go_github.com_multiplay_go-ts3:recipe} - ${go_github.com_naoina_go-stringutil:recipe} - ${go_github.com_nats-io_go-nats:recipe} - ${go_github.com_nats-io_nats:recipe} - ${go_github.com_nats-io_nuid:recipe} - ${go_github.com_nsqio_go-nsq:recipe} - ${go_github.com_opencontainers_runc:recipe} - ${go_github.com_opentracing-contrib_go-observer:recipe} - ${go_github.com_opentracing_opentracing-go:recipe} - ${go_github.com_openzipkin_zipkin-go-opentracing:recipe} - ${go_github.com_peterh_liner:recipe} - ${go_github.com_philhofer_fwd:recipe} - ${go_github.com_pierrec_lz4:recipe} - ${go_github.com_pierrec_xxHash:recipe} - ${go_github.com_pkg_errors:recipe} - ${go_github.com_pmezard_go-difflib:recipe} - ${go_github.com_prometheus_client_golang:recipe} - ${go_github.com_prometheus_client_model:recipe} - ${go_github.com_prometheus_common:recipe} - ${go_github.com_prometheus_procfs:recipe} - ${go_github.com_rcrowley_go-metrics:recipe} - ${go_github.com_retailnext_hllpp:recipe} - ${go_github.com_samuel_go-zookeeper:recipe} - ${go_github.com_satori_go.uuid:recipe} - ${go_github.com_shirou_gopsutil:recipe} - ${go_github.com_shirou_w32:recipe} - ${go_github.com_soniah_gosnmp:recipe} - ${go_github.com_sparrc_gdm:recipe} - ${go_github.com_streadway_amqp:recipe} - ${go_github.com_stretchr_objx:recipe} - ${go_github.com_stretchr_testify:recipe} - ${go_github.com_tidwall_gjson:recipe} - ${go_github.com_tidwall_match:recipe} - ${go_github.com_tinylib_msgp:recipe} - ${go_github.com_vjeantet_grok:recipe} - ${go_github.com_wvanbergen_kafka:recipe} - ${go_github.com_wvanbergen_kazoo-go:recipe} - ${go_github.com_xlab_treeprint:recipe} - ${go_github.com_yuin_gopher-lua:recipe} - ${go_github.com_zensqlmonitor_go-mssqldb:recipe} - ${go_go.uber.org_atomic:recipe} - ${go_go.uber.org_multierr:recipe} - ${go_go.uber.org_zap:recipe} - ${go_golang.org_x_crypto:recipe} - ${go_golang.org_x_net:recipe} - ${go_golang.org_x_sys:recipe} - ${go_golang.org_x_text:recipe} - ${go_golang.org_x_time:recipe} - ${go_golang.org_x_tools:recipe} - ${go_gopkg.in_asn1-ber.v1:recipe} - ${go_gopkg.in_fatih_pool.v2:recipe} - ${go_gopkg.in_fsnotify.v1:recipe} - ${go_gopkg.in_gorethink_gorethink.v3:recipe} - ${go_gopkg.in_ldap.v2:recipe} - ${go_gopkg.in_mgo.v2:recipe} - ${go_gopkg.in_olivere_elastic.v5:recipe} - ${go_gopkg.in_tomb.v1:recipe} - ${go_gopkg.in_yaml.v2:recipe} -[go_collectd.org] +[go_github.com_golang_dep] <= go-git-package -go.importpath = collectd.org -repository = https://github.com/collectd/go-collectd -revision = v0.3.0-17-g606bd390f3 - -[go_github.com_BurntSushi_toml] -<= go-git-package -go.importpath = github.com/BurntSushi/toml -repository = https://github.com/BurntSushi/toml -revision = v0.2.0-45-ga368813c5e - -[go_github.com_Microsoft_go-winio] -<= go-git-package -go.importpath = github.com/Microsoft/go-winio -repository = https://github.com/Microsoft/go-winio -revision = ce2922f643 - -[go_github.com_RoaringBitmap_roaring] -<= go-git-package -go.importpath = github.com/RoaringBitmap/roaring -repository = https://github.com/RoaringBitmap/roaring -revision = v0.2.8-174-g0a6691af7c - -[go_github.com_Shopify_sarama] -<= go-git-package -go.importpath = github.com/Shopify/sarama -repository = https://github.com/Shopify/sarama -revision = 3b1b38866a - -[go_github.com_Sirupsen_logrus] -<= go-git-package -go.importpath = github.com/Sirupsen/logrus -repository = https://github.com/Sirupsen/logrus -revision = 61e43dc76f - -[go_github.com_StackExchange_wmi] -<= go-git-package -go.importpath = github.com/StackExchange/wmi -repository = https://github.com/StackExchange/wmi -revision = f3e2bae1e0 - -[go_github.com_aerospike_aerospike-client-go] -<= go-git-package -go.importpath = github.com/aerospike/aerospike-client-go -repository = https://github.com/aerospike/aerospike-client-go -revision = v1.6.4-277-g95e1ad7791 - -[go_github.com_amir_raidman] -<= go-git-package -go.importpath = github.com/amir/raidman -repository = https://github.com/amir/raidman -revision = c74861fe6a - -[go_github.com_apache_thrift] -<= go-git-package -go.importpath = github.com/apache/thrift -repository = https://github.com/apache/thrift -revision = 4aaa92ece8 - -[go_github.com_aws_aws-sdk-go] -<= go-git-package -go.importpath = github.com/aws/aws-sdk-go -repository = https://github.com/aws/aws-sdk-go -revision = c861d27d03 - -[go_github.com_beorn7_perks] -<= go-git-package -go.importpath = github.com/beorn7/perks -repository = https://github.com/beorn7/perks -revision = 4c0e84591b - -[go_github.com_bmizerany_pat] -<= go-git-package -go.importpath = github.com/bmizerany/pat -repository = https://github.com/bmizerany/pat -revision = 6226ea591a - -[go_github.com_boltdb_bolt] -<= go-git-package -go.importpath = github.com/boltdb/bolt -repository = https://github.com/boltdb/bolt -revision = 9da3174536 - -[go_github.com_bsm_sarama-cluster] -<= go-git-package -go.importpath = github.com/bsm/sarama-cluster -repository = https://github.com/bsm/sarama-cluster -revision = v1.0.0-164-gabf039439f - -[go_github.com_cenkalti_backoff] -<= go-git-package -go.importpath = github.com/cenkalti/backoff -repository = https://github.com/cenkalti/backoff -revision = b02f2bbce1 - -[go_github.com_cespare_xxhash] -<= go-git-package -go.importpath = github.com/cespare/xxhash -repository = https://github.com/cespare/xxhash -revision = e4e2bd419c - -[go_github.com_couchbase_go-couchbase] -<= go-git-package -go.importpath = github.com/couchbase/go-couchbase -repository = https://github.com/couchbase/go-couchbase -revision = bfe555a140 - -[go_github.com_couchbase_gomemcached] -<= go-git-package -go.importpath = github.com/couchbase/gomemcached -repository = https://github.com/couchbase/gomemcached -revision = 4a25d2f4e1 - -[go_github.com_couchbase_goutils] -<= go-git-package -go.importpath = github.com/couchbase/goutils -repository = https://github.com/couchbase/goutils -revision = 5823a0cbaa - -[go_github.com_davecgh_go-spew] -<= go-git-package -go.importpath = github.com/davecgh/go-spew -repository = https://github.com/davecgh/go-spew -revision = v1.1.0-9-gecdeabc654 - -[go_github.com_dgrijalva_jwt-go] -<= go-git-package -go.importpath = github.com/dgrijalva/jwt-go -repository = https://github.com/dgrijalva/jwt-go -revision = dbeaa9332f - -[go_github.com_dgryski_go-bitstream] -<= go-git-package -go.importpath = github.com/dgryski/go-bitstream -repository = https://github.com/dgryski/go-bitstream -revision = 7d46cd22db - -[go_github.com_docker_docker] -<= go-git-package -go.importpath = github.com/docker/docker -repository = https://github.com/docker/docker -revision = v17.03.2-ce-0-gf5ec1e2936 - -[go_github.com_docker_go-connections] -<= go-git-package -go.importpath = github.com/docker/go-connections -repository = https://github.com/docker/go-connections -revision = 990a1a1a70 - -[go_github.com_eapache_go-resiliency] -<= go-git-package -go.importpath = github.com/eapache/go-resiliency -repository = https://github.com/eapache/go-resiliency -revision = b86b1ec0dd - -[go_github.com_eapache_go-xerial-snappy] -<= go-git-package -go.importpath = github.com/eapache/go-xerial-snappy -repository = https://github.com/eapache/go-xerial-snappy -revision = bb955e01b9 - -[go_github.com_eapache_queue] -<= go-git-package -go.importpath = github.com/eapache/queue -repository = https://github.com/eapache/queue -revision = 44cc805cf1 - -[go_github.com_eclipse_paho.mqtt.golang] -<= go-git-package -go.importpath = github.com/eclipse/paho.mqtt.golang -repository = https://github.com/eclipse/paho.mqtt.golang -revision = d4f545eb10 - -[go_github.com_glycerine_go-unsnap-stream] -<= go-git-package -go.importpath = github.com/glycerine/go-unsnap-stream -repository = https://github.com/glycerine/go-unsnap-stream -revision = 62a9a9eb44 - -[go_github.com_go-ini_ini] -<= go-git-package -go.importpath = github.com/go-ini/ini -repository = https://github.com/go-ini/ini -revision = 9144852efb - -[go_github.com_go-logfmt_logfmt] -<= go-git-package -go.importpath = github.com/go-logfmt/logfmt -repository = https://github.com/go-logfmt/logfmt -revision = v0.3.0-0-g390ab7935e - -[go_github.com_go-ole_go-ole] -<= go-git-package -go.importpath = github.com/go-ole/go-ole -repository = https://github.com/go-ole/go-ole -revision = be49f7c077 - -[go_github.com_go-sql-driver_mysql] -<= go-git-package -go.importpath = github.com/go-sql-driver/mysql -repository = https://github.com/go-sql-driver/mysql -revision = v1.0-470-g2e00b5cd70 - -[go_github.com_gobwas_glob] -<= go-git-package -go.importpath = github.com/gobwas/glob -repository = https://github.com/gobwas/glob -revision = v0.2.2-0-gbea32b9cd2 - -[go_github.com_gogo_protobuf] -<= go-git-package -go.importpath = github.com/gogo/protobuf -repository = https://github.com/gogo/protobuf -revision = 160de10b25 - -[go_github.com_golang_protobuf] -<= go-git-package -go.importpath = github.com/golang/protobuf -repository = https://github.com/golang/protobuf -revision = 1e59b77b52 - -[go_github.com_golang_snappy] -<= go-git-package -go.importpath = github.com/golang/snappy -repository = https://github.com/golang/snappy -revision = 553a641470 - -[go_github.com_google_go-cmp] -<= go-git-package -go.importpath = github.com/google/go-cmp -repository = https://github.com/google/go-cmp -revision = f94e52cad9 - -[go_github.com_gorilla_mux] -<= go-git-package -go.importpath = github.com/gorilla/mux -repository = https://github.com/gorilla/mux -revision = 392c28fe23 +go.importpath = github.com/golang/dep +repository = https://github.com/golang/dep +revision = 1f7c19e5f5 [go_github.com_grafana_grafana] <= go-git-package go.importpath = github.com/grafana/grafana repository = https://github.com/grafana/grafana -revision = v4.6.0-beta1-1360-g9606a34e0a - -[go_github.com_hailocab_go-hostpool] -<= go-git-package -go.importpath = github.com/hailocab/go-hostpool -repository = https://github.com/hailocab/go-hostpool -revision = e80d13ce29 +revision = v6.3.0-0-g830da0fda0 -[go_github.com_hashicorp_consul] +[go_github.com_grafana_loki] <= go-git-package -go.importpath = github.com/hashicorp/consul -repository = https://github.com/hashicorp/consul -revision = v0.7.3-35-g63d2fc6823 +go.importpath = github.com/grafana/loki +repository = https://github.com/grafana/loki +revision = v0.3.0-0-gc673380bb3 [go_github.com_influxdata_influxdb] <= go-git-package go.importpath = github.com/influxdata/influxdb repository = https://github.com/influxdata/influxdb -revision = v1.4.0rc0-328-g938db68198 - -[go_github.com_influxdata_influxql] -<= go-git-package -go.importpath = github.com/influxdata/influxql -repository = https://github.com/influxdata/influxql -revision = 851636b092 - -[go_github.com_influxdata_tail] -<= go-git-package -go.importpath = github.com/influxdata/tail -repository = https://github.com/influxdata/tail -revision = v0-95-ga395bf99fe +revision = v1.7.8rc1-0-ga7afa2ecf1 [go_github.com_influxdata_telegraf] <= go-git-package go.importpath = github.com/influxdata/telegraf repository = https://github.com/influxdata/telegraf -revision = 1.5.0-rc1-73-g90b6b760d1 - -[go_github.com_influxdata_toml] -<= go-git-package -go.importpath = github.com/influxdata/toml -repository = https://github.com/influxdata/toml -revision = 5d1d907f22 - -[go_github.com_influxdata_usage-client] -<= go-git-package -go.importpath = github.com/influxdata/usage-client -repository = https://github.com/influxdata/usage-client -revision = 6d38953763 - -[go_github.com_influxdata_wlog] -<= go-git-package -go.importpath = github.com/influxdata/wlog -repository = https://github.com/influxdata/wlog -revision = 7c63b0a71e - -[go_github.com_influxdata_yamux] -<= go-git-package -go.importpath = github.com/influxdata/yamux -repository = https://github.com/influxdata/yamux -revision = 1f58ded512 - -[go_github.com_influxdata_yarpc] -<= go-git-package -go.importpath = github.com/influxdata/yarpc -repository = https://github.com/influxdata/yarpc -revision = fdd7e84bf3 - -[go_github.com_jackc_pgx] -<= go-git-package -go.importpath = github.com/jackc/pgx -repository = https://github.com/jackc/pgx -revision = 63f58fd32e - -[go_github.com_jmespath_go-jmespath] -<= go-git-package -go.importpath = github.com/jmespath/go-jmespath -repository = https://github.com/jmespath/go-jmespath -revision = 0.2.2-14-gbd40a432e4 - -[go_github.com_jwilder_encoding] -<= go-git-package -go.importpath = github.com/jwilder/encoding -repository = https://github.com/jwilder/encoding -revision = b4e1701a28 - -[go_github.com_kardianos_govendor] -<= go-git-package -go.importpath = github.com/kardianos/govendor -repository = https://github.com/kardianos/govendor -revision = 274337c49c - -[go_github.com_kardianos_osext] -<= go-git-package -go.importpath = github.com/kardianos/osext -repository = https://github.com/kardianos/osext -revision = c2c54e542f - -[go_github.com_kardianos_service] -<= go-git-package -go.importpath = github.com/kardianos/service -repository = https://github.com/kardianos/service -revision = 6d3a0ee7d3 - -[go_github.com_kballard_go-shellquote] -<= go-git-package -go.importpath = github.com/kballard/go-shellquote -repository = https://github.com/kballard/go-shellquote -revision = d8ec1a69a2 - -[go_github.com_matttproud_golang_protobuf_extensions] -<= go-git-package -go.importpath = github.com/matttproud/golang_protobuf_extensions -repository = https://github.com/matttproud/golang_protobuf_extensions -revision = v1.0.0-2-gc12348ce28 - -[go_github.com_miekg_dns] -<= go-git-package -go.importpath = github.com/miekg/dns -repository = https://github.com/miekg/dns -revision = 99f84ae56e - -[go_github.com_mitchellh_mapstructure] -<= go-git-package -go.importpath = github.com/mitchellh/mapstructure -repository = https://github.com/mitchellh/mapstructure -revision = d0303fe809 - -[go_github.com_multiplay_go-ts3] -<= go-git-package -go.importpath = github.com/multiplay/go-ts3 -repository = https://github.com/multiplay/go-ts3 -revision = 07477f49b8 - -[go_github.com_naoina_go-stringutil] -<= go-git-package -go.importpath = github.com/naoina/go-stringutil -repository = https://github.com/naoina/go-stringutil -revision = 6b638e95a3 - -[go_github.com_nats-io_go-nats] -<= go-git-package -go.importpath = github.com/nats-io/go-nats -repository = https://github.com/nats-io/go-nats -revision = v1.2.0-73-gea9585611a - -[go_github.com_nats-io_nats] -<= go-git-package -go.importpath = github.com/nats-io/nats -repository = https://github.com/nats-io/nats -revision = v1.2.0-73-gea9585611a - -[go_github.com_nats-io_nuid] -<= go-git-package -go.importpath = github.com/nats-io/nuid -repository = https://github.com/nats-io/nuid -revision = 289cccf02c - -[go_github.com_nsqio_go-nsq] -<= go-git-package -go.importpath = github.com/nsqio/go-nsq -repository = https://github.com/nsqio/go-nsq -revision = v1.0.7-0-geee57a3ac4 - -[go_github.com_opencontainers_runc] -<= go-git-package -go.importpath = github.com/opencontainers/runc -repository = https://github.com/opencontainers/runc -revision = v0.0.5-426-g89ab7f2ccc - -[go_github.com_opentracing-contrib_go-observer] -<= go-git-package -go.importpath = github.com/opentracing-contrib/go-observer -repository = https://github.com/opentracing-contrib/go-observer -revision = a52f234244 - -[go_github.com_opentracing_opentracing-go] -<= go-git-package -go.importpath = github.com/opentracing/opentracing-go -repository = https://github.com/opentracing/opentracing-go -revision = 1361b9cd60 - -[go_github.com_openzipkin_zipkin-go-opentracing] -<= go-git-package -go.importpath = github.com/openzipkin/zipkin-go-opentracing -repository = https://github.com/openzipkin/zipkin-go-opentracing -revision = 1cafbdfde9 - -[go_github.com_peterh_liner] -<= go-git-package -go.importpath = github.com/peterh/liner -repository = https://github.com/peterh/liner -revision = 3681c2a912 - -[go_github.com_philhofer_fwd] -<= go-git-package -go.importpath = github.com/philhofer/fwd -repository = https://github.com/philhofer/fwd -revision = bb6d471dc9 - -[go_github.com_pierrec_lz4] -<= go-git-package -go.importpath = github.com/pierrec/lz4 -repository = https://github.com/pierrec/lz4 -revision = v1.0-0-g5c9560bfa9 - -[go_github.com_pierrec_xxHash] -<= go-git-package -go.importpath = github.com/pierrec/xxHash -repository = https://github.com/pierrec/xxHash -revision = 5a004441f8 - -[go_github.com_pkg_errors] -<= go-git-package -go.importpath = github.com/pkg/errors -repository = https://github.com/pkg/errors -revision = v0.8.0-0-g645ef00459 - -[go_github.com_pmezard_go-difflib] -<= go-git-package -go.importpath = github.com/pmezard/go-difflib -repository = https://github.com/pmezard/go-difflib -revision = v1.0.0-0-g792786c740 - -[go_github.com_prometheus_client_golang] -<= go-git-package -go.importpath = github.com/prometheus/client_golang -repository = https://github.com/prometheus/client_golang -revision = v0.9.0-pre1-21-g180b8fdc22 - -[go_github.com_prometheus_client_model] -<= go-git-package -go.importpath = github.com/prometheus/client_model -repository = https://github.com/prometheus/client_model -revision = model-0.0.2-16-g99fa1f4be8 - -[go_github.com_prometheus_common] -<= go-git-package -go.importpath = github.com/prometheus/common -repository = https://github.com/prometheus/common -revision = 89604d1970 - -[go_github.com_prometheus_procfs] -<= go-git-package -go.importpath = github.com/prometheus/procfs -repository = https://github.com/prometheus/procfs -revision = b15cd069a8 - -[go_github.com_rcrowley_go-metrics] -<= go-git-package -go.importpath = github.com/rcrowley/go-metrics -repository = https://github.com/rcrowley/go-metrics -revision = 1f30fe9094 - -[go_github.com_retailnext_hllpp] -<= go-git-package -go.importpath = github.com/retailnext/hllpp -repository = https://github.com/retailnext/hllpp -revision = v1.0.0-2-g6e8b6d3944 - -[go_github.com_samuel_go-zookeeper] -<= go-git-package -go.importpath = github.com/samuel/go-zookeeper -repository = https://github.com/samuel/go-zookeeper -revision = 1d7be4effb - -[go_github.com_satori_go.uuid] -<= go-git-package -go.importpath = github.com/satori/go.uuid -repository = https://github.com/satori/go.uuid -revision = v1.1.0-8-g5bf94b69c6 - -[go_github.com_shirou_gopsutil] -<= go-git-package -go.importpath = github.com/shirou/gopsutil -repository = https://github.com/shirou/gopsutil -revision = v2.16.10-243-g384a55110a - -[go_github.com_shirou_w32] -<= go-git-package -go.importpath = github.com/shirou/w32 -repository = https://github.com/shirou/w32 -revision = 3c9377fc67 - -[go_github.com_soniah_gosnmp] -<= go-git-package -go.importpath = github.com/soniah/gosnmp -repository = https://github.com/soniah/gosnmp -revision = v1.0-204-g5ad50dc75a - -[go_github.com_sparrc_gdm] -<= go-git-package -go.importpath = github.com/sparrc/gdm -repository = https://github.com/sparrc/gdm -revision = 1.4-4-g81089dabfa - -[go_github.com_streadway_amqp] -<= go-git-package -go.importpath = github.com/streadway/amqp -repository = https://github.com/streadway/amqp -revision = 63795daa9a - -[go_github.com_stretchr_objx] -<= go-git-package -go.importpath = github.com/stretchr/objx -repository = https://github.com/stretchr/objx -revision = 1a9d0bb9f5 - -[go_github.com_stretchr_testify] -<= go-git-package -go.importpath = github.com/stretchr/testify -repository = https://github.com/stretchr/testify -revision = v1.0-187-g4d4bfba8f1 - -[go_github.com_tidwall_gjson] -<= go-git-package -go.importpath = github.com/tidwall/gjson -repository = https://github.com/tidwall/gjson -revision = 0623bd8fbd - -[go_github.com_tidwall_match] -<= go-git-package -go.importpath = github.com/tidwall/match -repository = https://github.com/tidwall/match -revision = 173748da73 - -[go_github.com_tinylib_msgp] -<= go-git-package -go.importpath = github.com/tinylib/msgp -repository = https://github.com/tinylib/msgp -revision = 428e467e72 - -[go_github.com_vjeantet_grok] -<= go-git-package -go.importpath = github.com/vjeantet/grok -repository = https://github.com/vjeantet/grok -revision = d73e972b60 - -[go_github.com_wvanbergen_kafka] -<= go-git-package -go.importpath = github.com/wvanbergen/kafka -repository = https://github.com/wvanbergen/kafka -revision = bc265fedb9 - -[go_github.com_wvanbergen_kazoo-go] -<= go-git-package -go.importpath = github.com/wvanbergen/kazoo-go -repository = https://github.com/wvanbergen/kazoo-go -revision = 9689573521 - -[go_github.com_xlab_treeprint] -<= go-git-package -go.importpath = github.com/xlab/treeprint -repository = https://github.com/xlab/treeprint -revision = 06dfc6fa17 - -[go_github.com_yuin_gopher-lua] -<= go-git-package -go.importpath = github.com/yuin/gopher-lua -repository = https://github.com/yuin/gopher-lua -revision = 66c871e454 - -[go_github.com_zensqlmonitor_go-mssqldb] -<= go-git-package -go.importpath = github.com/zensqlmonitor/go-mssqldb -repository = https://github.com/zensqlmonitor/go-mssqldb -revision = ffe5510c6f - -[go_go.uber.org_atomic] -<= go-git-package -go.importpath = go.uber.org/atomic -repository = https://github.com/uber-go/atomic -revision = v1.3.1-0-g8474b86a5a - -[go_go.uber.org_multierr] -<= go-git-package -go.importpath = go.uber.org/multierr -repository = https://github.com/uber-go/multierr -revision = v1.1.0-1-gfb7d312c2c - -[go_go.uber.org_zap] -<= go-git-package -go.importpath = go.uber.org/zap -repository = https://github.com/uber-go/zap -revision = v1.7.1-11-gf85c78b1dd - -[go_golang.org_x_crypto] -<= go-git-package -go.importpath = golang.org/x/crypto -repository = https://go.googlesource.com/crypto -revision = b3c9a1d25c - -[go_golang.org_x_net] -<= go-git-package -go.importpath = golang.org/x/net -repository = https://go.googlesource.com/net -revision = ab555f366c - -[go_golang.org_x_sys] -<= go-git-package -go.importpath = golang.org/x/sys -repository = https://go.googlesource.com/sys -revision = 810d700034 - -[go_golang.org_x_text] -<= go-git-package -go.importpath = golang.org/x/text -repository = https://go.googlesource.com/text -revision = e19ae14969 - -[go_golang.org_x_time] -<= go-git-package -go.importpath = golang.org/x/time -repository = https://go.googlesource.com/time -revision = 6dc17368e0 - -[go_golang.org_x_tools] -<= go-git-package -go.importpath = golang.org/x/tools -repository = https://go.googlesource.com/tools -revision = fbec762f83 - -[go_gopkg.in_asn1-ber.v1] -<= go-git-package -go.importpath = gopkg.in/asn1-ber.v1 -repository = https://gopkg.in/asn1-ber.v1 -revision = 4e86f43671 - -[go_gopkg.in_fatih_pool.v2] -<= go-git-package -go.importpath = gopkg.in/fatih/pool.v2 -repository = https://gopkg.in/fatih/pool.v2 -revision = 6e328e6789 - -[go_gopkg.in_fsnotify.v1] -<= go-git-package -go.importpath = gopkg.in/fsnotify.v1 -repository = https://gopkg.in/fsnotify.v1 -revision = a8a77c9133 - -[go_gopkg.in_gorethink_gorethink.v3] -<= go-git-package -go.importpath = gopkg.in/gorethink/gorethink.v3 -repository = https://gopkg.in/gorethink/gorethink.v3 -revision = v2.2.1-35-g7ab832f7b6 - -[go_gopkg.in_ldap.v2] -<= go-git-package -go.importpath = gopkg.in/ldap.v2 -repository = https://gopkg.in/ldap.v2 -revision = 8168ee085e - -[go_gopkg.in_mgo.v2] -<= go-git-package -go.importpath = gopkg.in/mgo.v2 -repository = https://gopkg.in/mgo.v2 -revision = 3f83fa5005 - -[go_gopkg.in_olivere_elastic.v5] -<= go-git-package -go.importpath = gopkg.in/olivere/elastic.v5 -repository = https://gopkg.in/olivere/elastic.v5 -revision = v5.0.41-0-g3113f9b9ad - -[go_gopkg.in_tomb.v1] -<= go-git-package -go.importpath = gopkg.in/tomb.v1 -repository = https://gopkg.in/tomb.v1 -revision = dd632973f1 - -[go_gopkg.in_yaml.v2] -<= go-git-package -go.importpath = gopkg.in/yaml.v2 -repository = https://gopkg.in/yaml.v2 -revision = 4c78c975fe +revision = 1.11.1-0-gf91de60d37 diff --git a/software/grafana/grafana-provisioning-config-file.cfg.in b/software/grafana/grafana-provisioning-config-file.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..6d9e9e683262871ac48167430852a89a35a7bc46 --- /dev/null +++ b/software/grafana/grafana-provisioning-config-file.cfg.in @@ -0,0 +1,24 @@ +# https://grafana.com/docs/administration/provisioning/#example-datasource-config-file + +apiVersion: 1 + +datasources: +- name: telegraf + type: influxdb + access: proxy + url: {{ influxdb['url'] }} + user: {{ influxdb['auth-username'] }} + database: telegraf + isDefault: true + jsonData: + tlsSkipVerify: true + secureJsonData: + password: {{ influxdb['auth-password'] }} + version: 1 + editable: false +- name: loki + type: loki + access: proxy + url: {{ loki['url'] }} + version: 1 + editable: false diff --git a/software/grafana/instance-input-schema.json b/software/grafana/instance-input-schema.json index 357a6f53218ceda6b2ceee7546b9eb1dcb68596a..fc0e306d39eb86a3c9d635e3755132d6ba0ff1c9 100644 --- a/software/grafana/instance-input-schema.json +++ b/software/grafana/instance-input-schema.json @@ -28,6 +28,11 @@ "description": "Name used in From: header of emails", "default": "Grafana", "type": "string" + }, + "promtail-extra-scrape-config": { + "description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)", + "default": "", + "type": "string" } } } diff --git a/software/grafana/instance.cfg.in b/software/grafana/instance.cfg.in index ed2435f70a91a405e9038f312bf089625ab8d6d3..fe2ef8c63c3ee0d1c21a59528ac3632bde9ab228 100644 --- a/software/grafana/instance.cfg.in +++ b/software/grafana/instance.cfg.in @@ -3,8 +3,6 @@ parts = promises publish-connection-parameter -extends = {{ monitor_template }} - eggs-directory = {{ buildout['eggs-directory'] }} develop-eggs-directory = {{ buildout['develop-eggs-directory'] }} offline = true @@ -35,17 +33,21 @@ etc = ${:home}/etc var = ${:home}/var srv = ${:home}/srv service = ${:etc}/service +promise = ${:etc}/promise influxdb-data-dir = ${:srv}/influxdb grafana-dir = ${:srv}/grafana grafana-data-dir = ${:grafana-dir}/data grafana-logs-dir = ${:var}/log grafana-plugins-dir = ${:grafana-dir}/plugins grafana-provisioning-config-dir = ${:grafana-dir}/provisioning-config -grafana-provisioning-datasources = ${:grafana-provisioning-config-dir}/datasources -grafana-provisioning-dashboards = ${:grafana-provisioning-config-dir}/dashboards +grafana-provisioning-datasources-dir = ${:grafana-provisioning-config-dir}/datasources +grafana-provisioning-dashboards-dir = ${:grafana-provisioning-config-dir}/dashboards telegraf-dir = ${:srv}/telegraf telegraf-extra-config-dir = ${:telegraf-dir}/extra-config - +loki-dir = ${:srv}/loki +loki-storage-boltdb-dir = ${:loki-dir}/index/ +loki-storage-filesystem-dir = ${:loki-dir}/chunks/ +promtail-dir = ${:srv}/promtail # macros [generate-certificate] @@ -71,11 +73,14 @@ mode = 0644 extensions = jinja2.ext.do [check-port-listening-promise] -<= monitor-base-promise -module = check_port_listening -name = ${:_buildout_section_name_}.py - +recipe = slapos.cookbook:check_port_listening +path = ${directory:promise}/${:_buildout_section_name_} +[check-url-available-promise] +recipe = slapos.cookbook:check_url_available +path = ${directory:promise}/${:_buildout_section_name_} +dash_path = {{ dash_bin }} +curl_path = {{ curl_bin }} [influxdb] ipv6 = ${instance-parameter:ipv6-random} @@ -112,27 +117,27 @@ username = influxdb [influxdb-listen-promise] <= check-port-listening-promise -config-hostname = ${influxdb:ipv6} -config-port = ${influxdb:http-port} +hostname = ${influxdb:ipv6} +port = ${influxdb:http-port} [influxdb-password-promise] -<= monitor-base-promise -module = check_command_execute -name = ${:_buildout_section_name_}.py -config-command = +recipe = slapos.cookbook:wrapper +command-line = {{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES" +wrapper-path = ${directory:promise}/${:_buildout_section_name_} [grafana] ipv6 = ${instance-parameter:ipv6-random} -port = 8080 +port = 8180 url = https://[${:ipv6}]:${:port} data-dir = ${directory:grafana-data-dir} logs-dir = ${directory:grafana-logs-dir} plugins-dir = ${directory:grafana-plugins-dir} provisioning-config-dir = ${directory:grafana-provisioning-config-dir} +provisioning-datasources-dir = ${directory:grafana-provisioning-datasources-dir} admin-user = ${grafana-password:username} admin-password = ${grafana-password:passwd} secret-key = ${grafana-secret-key:passwd} @@ -160,11 +165,20 @@ context = section grafana grafana section apache_frontend apache-frontend key slapparameter_dict slap-configuration:configuration +depends = + ${grafana-provisioning-config-file:rendered} + +[grafana-provisioning-config-file] +<= config-file +rendered = ${grafana:provisioning-datasources-dir}/datasource.yaml +context = + section influxdb influxdb + section loki loki [grafana-listen-promise] <= check-port-listening-promise -config-hostname= ${grafana:ipv6} -config-port = ${grafana:port} +hostname= ${grafana:ipv6} +port = ${grafana:port} @@ -183,6 +197,53 @@ context = section telegraf telegraf +[loki] +recipe = slapos.cookbook:wrapper +command-line = + bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ loki_bin }} -config.file=${loki-config-file:rendered}' +wrapper-path = ${directory:service}/loki + +storage-boltdb-dir = ${directory:loki-storage-boltdb-dir} +storage-filesystem-dir = ${directory:loki-storage-filesystem-dir} +ip = ${instance-parameter:ipv4-random} +port = 3100 +url = http://${:ip}:${:port} + + +[loki-config-file] +<= config-file +context = + section loki loki + +[loki-listen-promise] +<= check-url-available-promise +url = ${loki:url}/ready + +[promtail] +recipe = slapos.cookbook:wrapper +command-line = + bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ promtail_bin }} -config.file=${promtail-config-file:rendered}' +wrapper-path = ${directory:service}/promtail + +dir = ${directory:promtail-dir} +http_port = 19080 +ip = ${instance-parameter:ipv4-random} +url = http://${:ip}:${:http_port} + +[promtail-config-file] +<= config-file +context = + section promtail promtail + section loki loki + key slapparameter_dict slap-configuration:configuration + +[promtail-listen-promise] +<= check-port-listening-promise +hostname= ${promtail:ip} +port = ${promtail:http_port} + + + [apache-frontend] <= slap-connection recipe = slapos.cookbook:requestoptional @@ -201,6 +262,8 @@ instance-promises = ${influxdb-listen-promise:path} ${influxdb-password-promise:wrapper-path} ${grafana-listen-promise:path} + ${loki-listen-promise:path} + ${promtail-listen-promise:path} [publish-connection-parameter] @@ -213,4 +276,6 @@ telegraf-extra-config-dir = ${telegraf:extra-config-dir} grafana-url = ${grafana:url} grafana-username = ${grafana:admin-user} grafana-password = ${grafana:admin-password} +loki-url = ${loki:url} +promtail-url = ${promtail:url} url = ${apache-frontend:connection-secure_access} diff --git a/software/grafana/loki-config-file.cfg.in b/software/grafana/loki-config-file.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..b63e6acbdf4dbf01df0bc684edf0b313609fcc2a --- /dev/null +++ b/software/grafana/loki-config-file.cfg.in @@ -0,0 +1,52 @@ +auth_enabled: false + +server: + http_listen_port: {{ loki['port'] }} + +ingester: + lifecycler: + address: {{ loki['ip'] }} + ring: + kvstore: + store: inmemory + replication_factor: 1 + chunk_idle_period: 15m + +schema_config: + configs: + - from: 2018-04-15 + store: boltdb + object_store: filesystem + schema: v9 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: {{ loki['storage-boltdb-dir'] }} + + filesystem: + directory: {{ loki['storage-filesystem-dir'] }} + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0 + +table_manager: + chunk_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + index_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + retention_deletes_enabled: false + retention_period: 0 diff --git a/software/grafana/promtail-config-file.cfg.in b/software/grafana/promtail-config-file.cfg.in new file mode 100644 index 0000000000000000000000000000000000000000..ac459f97216b50b4d85bc73ef30a0df9849a67ad --- /dev/null +++ b/software/grafana/promtail-config-file.cfg.in @@ -0,0 +1,23 @@ +# https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md + +server: + http_listen_port: {{ promtail['http_port'] }} + # XXX this external_url does not work ... promtail still listen on all IPs + external_url: {{ promtail['url'] }} + grpc_listen_port: 0 + +positions: + filename: {{Â promtail['dir'] }}/positions.yaml + +clients: + - url: {{ loki['url'] }}/api/prom/push + +scrape_configs: +- job_name: test + static_configs: + - targets: + - localhost + labels: + job: grafanalogs + __path__: ./var/log/*log +{{ slapparameter_dict.get('promtail-extra-scrape-config', '') }} diff --git a/software/grafana/software.cfg b/software/grafana/software.cfg index 47d9b895b52e7b6e9c2bef7ed4e2025424d525d7..52e6711e408475419874ab07a4eeb6cfe12c10a4 100644 --- a/software/grafana/software.cfg +++ b/software/grafana/software.cfg @@ -2,10 +2,11 @@ extends = ../../stack/slapos.cfg ../../stack/nodejs.cfg - ../../stack/monitor/buildout.cfg ../../component/make/buildout.cfg ../../component/golang/buildout.cfg ../../component/openssl/buildout.cfg + ../../component/curl/buildout.cfg + ../../component/dash/buildout.cfg buildout.hash.cfg gowork.cfg @@ -17,21 +18,24 @@ parts = influxdb-config-file telegraf-config-file grafana-config-file + grafana-provisioning-config-file + loki-config-file + promtail-config-file [nodejs] -<= nodejs-8.6.0 +<= nodejs-10.6.0 [yarn] # this could become a component, but it needs to be invoked from nodejs explicitly, # otherwise it uses system's nodejs recipe = slapos.recipe.build:download-unpacked -url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz -md5sum = db82fa09c996e9318f2f1d2ab99228f9 +url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz +md5sum = 46790033c23803387890f545e4040690 [gowork] -# All the softwares installed in the go work have "non standard" installation +# All the softwares installed in the go workspace have "non standard" installation # methods, so we install them in specific parts with custom commands. # They will be installed because they are dependencies of ${gowork.goinstall} install = @@ -41,24 +45,35 @@ influx-bin = ${:bin}/influx influxd-bin = ${:bin}/influxd grafana-bin = ${:bin}/grafana-server grafana-homepath = ${go_github.com_grafana_grafana:location} +loki-bin = ${:bin}/loki +promtail-bin = ${:bin}/promtail + +# use recent go +golang = ${golang1.12:location} +gcc-bin-directory = ${golang1.12:gcc-bin-directory} [gowork.goinstall] command = : -depends = +depends = ${influxdb-install:recipe} ${telegraf-install:recipe} ${grafana-install:recipe} + ${loki-install:recipe} + ${promtail-install:recipe} [influxdb-install] <= gowork.goinstall command = bash -c ". ${gowork:env.sh} && \ + go install -v github.com/golang/dep/cmd/dep && \ cd ${gowork:directory}/src/github.com/influxdata/influxdb && \ + dep ensure && \ go install ./cmd/..." update-command = [telegraf-install] <= gowork.goinstall command = bash -c ". ${gowork:env.sh} && \ + go install -v github.com/golang/dep/cmd/dep && \ cd ${gowork:directory}/src/github.com/influxdata/telegraf && \ ${make:location}/bin/make && cp telegraf ${gowork:bin}" @@ -70,12 +85,32 @@ update-command = command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \ . ${gowork:env.sh} && \ cd ${gowork:directory}/src/github.com/grafana/grafana && \ - ${gowork:golang}/bin/go run build.go setup && \ - ${gowork:golang}/bin/go run build.go build && \ + go run build.go setup && \ + go run build.go build && \ ${yarn:location}/bin/yarn install --pure-lockfile && \ ${nodejs:location}/bin/npm run build" update-command = +[loki-install] +<= gowork.goinstall +# loki also uses nodejs +command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \ + . ${gowork:env.sh} && \ + go install -v github.com/golang/dep/cmd/dep && \ + cd ${gowork:directory}/src/github.com/grafana/loki && \ + go install ./cmd/loki" +update-command = + +[promtail-install] +<= gowork.goinstall +# CGO_ENABLED is to disable systemd support (did not compile in my case) +command = bash -c "export CGO_ENABLED=0 && \ + . ${gowork:env.sh} && \ + go install -v github.com/golang/dep/cmd/dep && \ + cd ${gowork:directory}/src/github.com/grafana/loki && \ + go install ./cmd/promtail" +update-command = + [download-file-base] recipe = slapos.recipe.build:download @@ -92,6 +127,15 @@ mode = 0644 [grafana-config-file] <= download-file-base +[grafana-provisioning-config-file] +<= download-file-base + +[loki-config-file] +<= download-file-base + +[promtail-config-file] +<= download-file-base + [instance-profile] recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/${:filename} @@ -106,10 +150,13 @@ context = key influx_bin gowork:influx-bin key grafana_bin gowork:grafana-bin key grafana_homepath gowork:grafana-homepath - key monitor_template monitor2-template:rendered - + key loki_bin gowork:loki-bin + key promtail_bin gowork:promtail-bin + key curl_bin :curl-bin + key dash_bin :dash-bin +curl-bin = ${curl:location}/bin/curl +dash-bin = ${dash:location}/bin/dash [versions] slapos.recipe.template = 4.2 inotifyx = 0.2.2 - diff --git a/software/grafana/test/README.md b/software/grafana/test/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ce0777e422c4102fab63401292f7dd0e5a43b01d --- /dev/null +++ b/software/grafana/test/README.md @@ -0,0 +1 @@ +Tests for Grafana software release diff --git a/software/grafana/test/setup.py b/software/grafana/test/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..749a2e174800d52df46f81b93fb364d4ac020868 --- /dev/null +++ b/software/grafana/test/setup.py @@ -0,0 +1,53 @@ +############################################################################## +# +# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from setuptools import setup, find_packages + +version = '0.0.1.dev0' +name = 'slapos.test.grafana' +long_description = open("README.md").read() + +setup( + name=name, + version=version, + description="Test for SlapOS' Grafana", + long_description=long_description, + long_description_content_type='text/markdown', + maintainer="Nexedi", + maintainer_email="info@nexedi.com", + url="https://lab.nexedi.com/nexedi/slapos", + packages=find_packages(), + install_requires=[ + 'slapos.core', + 'slapos.libnetworkcache', + 'erp5.util', + 'requests', + 'supervisor', + 'psutil', + ], + zip_safe=True, + test_suite='test', +) diff --git a/software/grafana/test/test.py b/software/grafana/test/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3790e641523091e31c0bca99bf91dff701726a0c --- /dev/null +++ b/software/grafana/test/test.py @@ -0,0 +1,228 @@ +############################################################################## +# +# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import os +import textwrap +import logging +import tempfile +import time + +import requests + +from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass + + +setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( + os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) + + +class GrafanaTestCase(SlapOSInstanceTestCase): + """Base test case for grafana. + + Since the instances takes timte to start and stop, + we increate as lot the number of retries. + """ + report_max_retry = 30 + instance_max_retry = 30 + + +class TestGrafana(GrafanaTestCase): + def setUp(self): + self.grafana_url = self.computer_partition.getConnectionParameterDict( + )['grafana-url'] + + def test_grafana_available(self): + resp = requests.get(self.grafana_url, verify=False) + self.assertEqual(requests.codes.ok, resp.status_code) + + def test_grafana_api(self): + # check API is usable + api_org_url = '{self.grafana_url}/api/org'.format(**locals()) + resp = requests.get(api_org_url, verify=False) + self.assertEqual(requests.codes.unauthorized, resp.status_code) + + connection_params = self.computer_partition.getConnectionParameterDict() + resp = requests.get( + api_org_url, + verify=False, + auth=requests.auth.HTTPBasicAuth( + connection_params['grafana-username'], + connection_params['grafana-password'], + )) + self.assertEqual(requests.codes.ok, resp.status_code) + self.assertEqual(1, resp.json()['id']) + + def test_grafana_datasource_povisinonned(self): + # data sources are provisionned + connection_params = self.computer_partition.getConnectionParameterDict() + resp = requests.get( + '{self.grafana_url}/api/datasources'.format(**locals()), + verify=False, + auth=requests.auth.HTTPBasicAuth( + connection_params['grafana-username'], + connection_params['grafana-password'], + )) + self.assertEqual(requests.codes.ok, resp.status_code) + self.assertEqual( + sorted(['influxdb', 'loki']), + sorted([ds['type'] for ds in resp.json()])) + + +class TestInfluxDb(GrafanaTestCase): + def setUp(self): + self.influxdb_url = self.computer_partition.getConnectionParameterDict( + )['influxdb-url'] + + def test_influxdb_available(self): + ping_url = '{self.influxdb_url}/ping'.format(**locals()) + resp = requests.get(ping_url, verify=False) + self.assertEqual(requests.codes.no_content, resp.status_code) + + def test_influxdb_api(self): + query_url = '{self.influxdb_url}/query'.format(**locals()) + connection_params = self.computer_partition.getConnectionParameterDict() + + for i in range(10): + # retry, as it may take a little delay to create databases + resp = requests.get( + query_url, + verify=False, + params=dict( + q='SHOW DATABASES', + u=connection_params['influxdb-username'], + p=connection_params['influxdb-password'])) + self.assertEqual(requests.codes.ok, resp.status_code) + result, = resp.json()['results'] + if result['series'] and 'values' in result['series'][0]: + break + time.sleep(0.5 * i) + + self.assertIn( + [connection_params['influxdb-database']], result['series'][0]['values']) + + +class TestTelegraf(GrafanaTestCase): + def test_telegraf_running(self): + with self.slap.instance_supervisor_rpc as supervisor: + all_process_info = supervisor.getAllProcessInfo() + process_info, = [p for p in all_process_info if 'telegraf' in p['name']] + self.assertEqual('RUNNING', process_info['statename']) + + +class TestLoki(GrafanaTestCase): + @classmethod + def getInstanceParameterDict(cls): + cls._logfile = tempfile.NamedTemporaryFile(suffix='log') + return { + 'promtail-extra-scrape-config': + textwrap.dedent( + r''' + - job_name: {cls.__name__} + pipeline_stages: + - regex: + expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)" + - timestamp: + format: 2006-01-02T15:04:05Z00:00 + source: timestamp + - labels: + level: + name: + static_configs: + - targets: + - localhost + labels: + job: {cls.__name__} + __path__: {cls._logfile.name} + ''').format(**locals()) + } + + @classmethod + def tearDownClass(cls): + cls._logfile.close() + super(TestLoki, cls).tearDownClass() + + def setUp(self): + self.loki_url = self.computer_partition.getConnectionParameterDict( + )['loki-url'] + + def test_loki_available(self): + self.assertEqual( + requests.codes.ok, + requests.get('{self.loki_url}/ready'.format(**locals()), + verify=False).status_code) + + def test_log_ingested(self): + # create a logger logging to the file that we have + # configured in instance parameter. + test_logger = logging.getLogger(self.id()) + test_logger.setLevel(logging.INFO) + test_handler = logging.FileHandler(filename=self._logfile.name) + test_handler.setFormatter( + logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + test_logger.addHandler(test_handler) + test_logger.info("testing message") + test_logger.info("testing another message") + test_logger.warning("testing warn") + + # Check our messages have been ingested + # we retry a few times, because there's a short delay until messages are + # ingested and returned. + for i in range(10): + resp = requests.get( + '{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format( + **locals()), + verify=False).json() + if not resp: + time.sleep(0.5 * i) + continue + + warn_stream_list = [stream for stream in resp['streams'] if 'level="WARNING"' in stream['labels']] + self.assertEqual(1, len(warn_stream_list), resp['streams']) + warn_stream, = warn_stream_list + self.assertIn("testing warn", warn_stream['entries'][0]['line']) + + info_stream_list = [stream for stream in resp['streams'] if 'level="INFO"' in stream['labels']] + self.assertEqual(1, len(info_stream_list), resp['streams']) + info_stream, = info_stream_list + self.assertTrue( + [ + line for line in info_stream['entries'] + if "testing message" in line['line'] + ]) + self.assertTrue( + [ + line for line in info_stream['entries'] + if "testing another message" in line['line'] + ]) + # The labels we have configued are also available + resp = requests.get( + '{self.loki_url}/api/prom/label'.format(**locals()), + verify=False).json() + self.assertIn('level', resp['values']) + self.assertIn('name', resp['values']) diff --git a/software/slapos-sr-testing/buildout.hash.cfg b/software/slapos-sr-testing/buildout.hash.cfg index e992cd9561ba221199292a5c38bbf1465a29023d..3963f6d5f27577a9a634adfca91682c54e4077fe 100644 --- a/software/slapos-sr-testing/buildout.hash.cfg +++ b/software/slapos-sr-testing/buildout.hash.cfg @@ -15,4 +15,4 @@ [template] filename = instance.cfg -md5sum = 4664f7dae66d3f582e34cec2ca627501 +md5sum = 1cbab58e896ff63575f6a67db530d183 diff --git a/software/slapos-sr-testing/instance.cfg b/software/slapos-sr-testing/instance.cfg index 4fcd0e1d8a7c1e388e981f7f6fad166ccdcfec6f..3bb02d715a6704afa6c524d2f3367dab7ba8bb58 100644 --- a/software/slapos-sr-testing/instance.cfg +++ b/software/slapos-sr-testing/instance.cfg @@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin working-dir = $${buildout:directory}/tmp [test-list] -path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup} +path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup} [slapos-test-runner] recipe = slapos.cookbook:wrapper diff --git a/software/slapos-sr-testing/software.cfg b/software/slapos-sr-testing/software.cfg index 64eb19268523ada4c6acf73e9aeca3a0c7cd2100..11696756385455252e2584699f2bf5b85f2660a6 100644 --- a/software/slapos-sr-testing/software.cfg +++ b/software/slapos-sr-testing/software.cfg @@ -107,6 +107,11 @@ setup = ${slapos-repository:location}/software/turnserver/test/ egg = slapos.test.theia setup = ${slapos-repository:location}/software/theia/test/ +[slapos.test.grafana-setup] +<= setup-develop-egg +egg = slapos.test.grafana +setup = ${slapos-repository:location}/software/grafana/test/ + [slapos.core-repository] <= git-clone-repository repository = https://lab.nexedi.com/nexedi/slapos.core.git