Commit 63e03dad authored by Lin Jen-Shin's avatar Lin Jen-Shin

Make various trace methods take last_lines argument:

So that we could read last few lines rather than read
the entire file which could be huge.
parent 5869fb20
...@@ -43,7 +43,9 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -43,7 +43,9 @@ class Projects::BuildsController < Projects::ApplicationController
def trace def trace
respond_to do |format| respond_to do |format|
format.json do format.json do
render json: @build.trace_with_state(params[:state].presence).merge!(id: @build.id, status: @build.status) state = params[:state].presence
render json: @build.trace_with_state(state: state).
merge!(id: @build.id, status: @build.status)
end end
end end
end end
......
...@@ -128,13 +128,14 @@ module Ci ...@@ -128,13 +128,14 @@ module Ci
latest_builds.where('stage_idx < ?', stage_idx) latest_builds.where('stage_idx < ?', stage_idx)
end end
def trace_html def trace_html(**args)
trace_with_state[:html] || '' trace_with_state(**args)[:html] || ''
end end
def trace_with_state(state = nil) def trace_with_state(state: nil, last_lines: nil)
if trace.present? trace_ansi = trace(last_lines)
Ci::Ansi2html.convert(trace, state) if trace_ansi.present?
Ci::Ansi2html.convert(trace_ansi, state)
else else
{} {}
end end
...@@ -220,9 +221,10 @@ module Ci ...@@ -220,9 +221,10 @@ module Ci
raw_trace.present? raw_trace.present?
end end
def raw_trace def raw_trace(last_lines: nil)
if File.exist?(trace_file_path) if File.exist?(trace_file_path)
File.read(trace_file_path) Gitlab::Ci::TraceReader.new(trace_file_path).
read(last_lines: last_lines)
else else
# backward compatibility # backward compatibility
read_attribute :trace read_attribute :trace
...@@ -237,8 +239,8 @@ module Ci ...@@ -237,8 +239,8 @@ module Ci
project.ci_id && File.exist?(old_path_to_trace) project.ci_id && File.exist?(old_path_to_trace)
end end
def trace def trace(last_lines: nil)
result = raw_trace result = raw_trace(last_lines)
if project && result.present? && project.runners_token.present? if project && result.present? && project.runners_token.present?
result.gsub(project.runners_token, 'xxxxxx') result.gsub(project.runners_token, 'xxxxxx')
else else
......
module Gitlab
module Ci
# This was inspired from: http://stackoverflow.com/a/10219411/1520132
class TraceReader
BUFFER_SIZE = 4096
attr_accessor :path, :buffer_size
def initialize(new_path, buffer_size: BUFFER_SIZE)
self.path = new_path
self.buffer_size = Integer(buffer_size)
end
def read(last_lines: nil)
if last_lines
read_last_lines(last_lines)
else
File.read(path)
end
end
def read_last_lines(max_lines)
File.open(path) do |file|
chunks = []
pos = lines = 0
max = file.size
# We want an extra line to make sure fist line has full contents
while lines <= max_lines && pos < max
pos += buffer_size
buf = if pos <= max
file.seek(-pos, IO::SEEK_END)
file.read(buffer_size)
else # Reached the head, read only left
file.seek(0)
file.read(buffer_size - (pos - max))
end
lines += buf.count("\n")
chunks.unshift(buf)
end
chunks.join.lines.last(max_lines).join
end
end
end
end
end
require 'spec_helper'
describe Gitlab::Ci::TraceReader do
let(:path) { __FILE__ }
let(:lines) { File.readlines(path) }
let(:bytesize) { lines.sum(&:bytesize) }
it 'returns last few lines' do
10.times do
subject = build_subject
last_lines = random_lines
expected = lines.last(last_lines).join
expect(subject.read(last_lines: last_lines)).to eq(expected)
end
end
it 'returns everything if trying to get too many lines' do
expect(build_subject.read(last_lines: lines.size * 2)).to eq(lines.join)
end
it 'raises an error if not passing an integer for last_lines' do
expect do
build_subject.read(last_lines: lines)
end.to raise_error(ArgumentError)
end
def random_lines
Random.rand(lines.size) + 1
end
def random_buffer
Random.rand(bytesize) + 1
end
def build_subject
described_class.new(__FILE__, buffer_size: random_buffer)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment