Commit 13818932 authored by Jérome Perrin's avatar Jérome Perrin

software/grafana: version up

update softwares to:
  - grafana 7.4.3
  - telegrah 1.17.3
  - influxdb 1.8.4
  - loki current master ( 2.1.0 )

and components to:
  - golang 1.16
  - nodejs 14.16.0
  - yarn 1.22.10
  - python 3.7

This new loki version supports parsing multi-line logs, as demonstrated in
the updated test.

Since this version we also remove cache at the end of build, which
makes the software directory smaller and accessorily prevent test failures
during the ldd checks from checkSoftware with some binaries in the cache.
parent 20abfb99
...@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions. ...@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions.
See https://github.com/influxdata/telegraf to learn about plugins. See https://github.com/influxdata/telegraf to learn about plugins.
Useful plugins in this context are probably Useful plugins in this context are probably
[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec), [exec](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser) [logparser](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/logparser)
or or
[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http). [http](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/http).
Telegraf will save in the `telegraf` database from the embedded influxdb server. Telegraf will save in the `telegraf` database from the embedded influxdb server.
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 13f4f3806522b265dd1912ff169d146d md5sum = 12d9507c6dfb1ae9151a0bf7eb741060
[influxdb-config-file] [influxdb-config-file]
filename = influxdb-config-file.cfg.in filename = influxdb-config-file.cfg.in
......
...@@ -20,22 +20,22 @@ revision = 1f7c19e5f5 ...@@ -20,22 +20,22 @@ revision = 1f7c19e5f5
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/grafana go.importpath = github.com/grafana/grafana
repository = https://github.com/grafana/grafana repository = https://github.com/grafana/grafana
revision = v6.3.0-0-g830da0fda0 revision = v7.4.3-0-g010f20c1c8
[go_github.com_grafana_loki] [go_github.com_grafana_loki]
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/loki go.importpath = github.com/grafana/loki
repository = https://github.com/grafana/loki repository = https://github.com/grafana/loki
revision = v0.3.0-0-gc673380bb3 revision = v2.1.0-188-g877f524c36
[go_github.com_influxdata_influxdb] [go_github.com_influxdata_influxdb]
<= go-git-package <= go-git-package
go.importpath = github.com/influxdata/influxdb go.importpath = github.com/influxdata/influxdb
repository = https://github.com/influxdata/influxdb repository = https://github.com/influxdata/influxdb
revision = v1.7.8rc1-0-ga7afa2ecf1 revision = v1.8.4-0-gbc8ec4384e
[go_github.com_influxdata_telegraf] [go_github.com_influxdata_telegraf]
<= go-git-package <= go-git-package
go.importpath = github.com/influxdata/telegraf go.importpath = github.com/influxdata/telegraf
repository = https://github.com/influxdata/telegraf repository = https://github.com/influxdata/telegraf
revision = 1.11.1-0-gf91de60d37 revision = v1.17.3-0-g24a552b90b
...@@ -132,7 +132,7 @@ command-line = ...@@ -132,7 +132,7 @@ command-line =
bash -c "{{ influx_bin }} \ bash -c "{{ influx_bin }} \
-username ${influxdb:auth-username} \ -username ${influxdb:auth-username} \
-password ${influxdb:auth-password} \ -password ${influxdb:auth-password} \
-host ${influxdb:host} \ -host [${influxdb:host}] \
-port ${influxdb:http-port} \ -port ${influxdb:http-port} \
-unsafeSsl \ -unsafeSsl \
-ssl \ -ssl \
......
...@@ -22,12 +22,14 @@ parts = ...@@ -22,12 +22,14 @@ parts =
loki-config-file loki-config-file
promtail-config-file promtail-config-file
[python]
part = python3
[nodejs] [nodejs]
<= nodejs-10.6.0 <= nodejs-14.16.0
[yarn] [yarn]
<= yarn-1.16.0 <= yarn-1.22.10
[gowork] [gowork]
# All the softwares installed in the go workspace have "non standard" installation # All the softwares installed in the go workspace have "non standard" installation
...@@ -44,7 +46,7 @@ loki-bin = ${:bin}/loki ...@@ -44,7 +46,7 @@ loki-bin = ${:bin}/loki
promtail-bin = ${:bin}/promtail promtail-bin = ${:bin}/promtail
# use recent go # use recent go
golang = ${golang1.12:location} golang = ${golang1.16:location}
[gowork.goinstall] [gowork.goinstall]
command = : command = :
...@@ -60,8 +62,7 @@ depends = ...@@ -60,8 +62,7 @@ depends =
command = bash -c ". ${gowork:env.sh} && \ command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \ go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/influxdb && \ cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
dep ensure && \ go install ./..."
go install ./cmd/..."
update-command = update-command =
[telegraf-install] [telegraf-install]
...@@ -75,14 +76,17 @@ update-command = ...@@ -75,14 +76,17 @@ update-command =
[grafana-install] [grafana-install]
<= gowork.goinstall <= gowork.goinstall
# yarn and go run build.go needs our nodejs in $PATH command = bash -c ". ${gowork:env.sh} && \
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
cd ${gowork:directory}/src/github.com/grafana/grafana && \ cd ${gowork:directory}/src/github.com/grafana/grafana && \
go run build.go setup && \ go run build.go setup && \
go run build.go build && \ go run build.go build && \
${yarn:location}/bin/yarn install --pure-lockfile && \ ${yarn:location}/bin/yarn install --pure-lockfile && \
${nodejs:location}/bin/npm run build" ${yarn:location}/bin/yarn run build && \
${yarn:location}/bin/yarn run plugins:build-bundled && \
# Cleanup yarn and Cypress caches
rm -rf ${buildout:directory}/.cache/Cypress/ && \
rm -rf ${buildout:directory}/.cache/yarn/
"
update-command = update-command =
[loki-install] [loki-install]
......
...@@ -44,11 +44,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( ...@@ -44,11 +44,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
class GrafanaTestCase(SlapOSInstanceTestCase): class GrafanaTestCase(SlapOSInstanceTestCase):
"""Base test case for grafana. """Base test case for grafana.
Since the instances takes timte to start and stop, Since the instances takes time to start and stop,
we increate as lot the number of retries. we increase the number of retries.
""" """
instance_max_retry = 50
report_max_retry = 30 report_max_retry = 30
instance_max_retry = 30
class TestGrafana(GrafanaTestCase): class TestGrafana(GrafanaTestCase):
...@@ -144,21 +144,27 @@ class TestLoki(GrafanaTestCase): ...@@ -144,21 +144,27 @@ class TestLoki(GrafanaTestCase):
r''' r'''
- job_name: {cls.__name__} - job_name: {cls.__name__}
pipeline_stages: pipeline_stages:
- regex: - match:
expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)" selector: '{{job="{cls.__name__}"}}'
- timestamp: stages:
format: 2006-01-02T15:04:05Z00:00 - multiline:
source: timestamp firstline: '^\d{{4}}-\d{{2}}-\d{{2}}\s\d{{1,2}}\:\d{{2}}\:\d{{2}}\,\d{{3}}'
- labels: max_wait_time: 3s
level: - regex:
name: expression: '^(?P<timestamp>.*) - (?P<name>\S+) - (?P<level>\S+) - (?P<message>.*)'
- timestamp:
format: 2006-01-02T15:04:05Z00:00
source: timestamp
- labels:
level:
name:
static_configs: static_configs:
- targets: - targets:
- localhost - localhost
labels: labels:
job: {cls.__name__} job: {cls.__name__}
__path__: {cls._logfile.name} __path__: {cls._logfile.name}
''').format(**locals()) ''').format(**locals())
} }
@classmethod @classmethod
...@@ -180,6 +186,7 @@ class TestLoki(GrafanaTestCase): ...@@ -180,6 +186,7 @@ class TestLoki(GrafanaTestCase):
# create a logger logging to the file that we have # create a logger logging to the file that we have
# configured in instance parameter. # configured in instance parameter.
test_logger = logging.getLogger(self.id()) test_logger = logging.getLogger(self.id())
test_logger.propagate = False
test_logger.setLevel(logging.INFO) test_logger.setLevel(logging.INFO)
test_handler = logging.FileHandler(filename=self._logfile.name) test_handler = logging.FileHandler(filename=self._logfile.name)
test_handler.setFormatter( test_handler.setFormatter(
...@@ -189,16 +196,25 @@ class TestLoki(GrafanaTestCase): ...@@ -189,16 +196,25 @@ class TestLoki(GrafanaTestCase):
test_logger.info("testing message") test_logger.info("testing message")
test_logger.info("testing another message") test_logger.info("testing another message")
test_logger.warning("testing warn") test_logger.warning("testing warn")
# log an exception, which will be multi line in log file.
def nested1():
def nested2():
raise ValueError('boom')
nested2()
try:
nested1()
except ValueError:
test_logger.exception("testing exception")
# Check our messages have been ingested # Check our messages have been ingested
# we retry a few times, because there's a short delay until messages are # we retry a few times, because there's a short delay until messages are
# ingested and returned. # ingested and returned.
for i in range(10): for i in range(60):
resp = requests.get( resp = requests.get(
'{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format( '{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
**locals()), **locals()),
verify=False).json() verify=False).json()
if not resp: if len(resp.get('streams', [])) < 3:
time.sleep(0.5 * i) time.sleep(0.5 * i)
continue continue
...@@ -220,6 +236,15 @@ class TestLoki(GrafanaTestCase): ...@@ -220,6 +236,15 @@ class TestLoki(GrafanaTestCase):
line for line in info_stream['entries'] line for line in info_stream['entries']
if "testing another message" in line['line'] if "testing another message" in line['line']
]) ])
error_stream_list = [stream for stream in resp['streams'] if 'level="ERROR"' in stream['labels']]
self.assertEqual(1, len(error_stream_list), resp['streams'])
error_stream, = error_stream_list
line, = [line['line'] for line in error_stream['entries']]
# this entry is multi-line
self.assertIn('testing exception\nTraceback (most recent call last):\n', line)
self.assertIn('ValueError: boom', line)
# The labels we have configued are also available # The labels we have configued are also available
resp = requests.get( resp = requests.get(
'{self.loki_url}/api/prom/label'.format(**locals()), '{self.loki_url}/api/prom/label'.format(**locals()),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment