Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Boxiang Sun
gitlab-ce
Commits
3299680c
Commit
3299680c
authored
6 years ago
by
Rémy Coutable
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[CE] Port review apps file to CE
Signed-off-by:
Rémy Coutable
<
remy@rymai.me
>
parent
cb0f024c
No related merge requests found
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
459 additions
and
0 deletions
+459
-0
lib/quality/helm_client.rb
lib/quality/helm_client.rb
+47
-0
lib/quality/kubernetes_client.rb
lib/quality/kubernetes_client.rb
+32
-0
scripts/review_apps/automated_cleanup.rb
scripts/review_apps/automated_cleanup.rb
+109
-0
scripts/review_apps/review-apps.sh
scripts/review_apps/review-apps.sh
+184
-0
spec/lib/quality/helm_client_spec.rb
spec/lib/quality/helm_client_spec.rb
+62
-0
spec/lib/quality/kubernetes_client_spec.rb
spec/lib/quality/kubernetes_client_spec.rb
+25
-0
No files found.
lib/quality/helm_client.rb
0 → 100644
View file @
3299680c
# frozen_string_literal: true
require
'time'
require_relative
'../gitlab/popen'
unless
defined?
(
Gitlab
::
Popen
)
module
Quality
class
HelmClient
attr_reader
:namespace
Release
=
Struct
.
new
(
:name
,
:revision
,
:last_update
,
:status
,
:chart
,
:namespace
)
do
def
revision
@revision
||=
self
[
:revision
].
to_i
end
def
last_update
@last_update
||=
Time
.
parse
(
self
[
:last_update
])
end
end
def
initialize
(
namespace:
ENV
[
'KUBE_NAMESPACE'
])
@namespace
=
namespace
end
def
releases
(
args:
[])
command
=
[
'list'
,
%(--namespace "#{namespace}")
,
*
args
]
run_command
(
command
)
.
stdout
.
lines
.
select
{
|
line
|
line
.
include?
(
namespace
)
}
.
map
{
|
line
|
Release
.
new
(
*
line
.
split
(
/\t/
).
map
(
&
:strip
))
}
end
def
delete
(
release_name
:)
run_command
([
'delete'
,
'--purge'
,
release_name
])
end
private
def
run_command
(
command
)
final_command
=
[
'helm'
,
*
command
].
join
(
' '
)
puts
"Running command: `
#{
final_command
}
`"
# rubocop:disable Rails/Output
Gitlab
::
Popen
.
popen_with_detail
([
final_command
])
end
end
end
This diff is collapsed.
Click to expand it.
lib/quality/kubernetes_client.rb
0 → 100644
View file @
3299680c
# frozen_string_literal: true
require_relative
'../gitlab/popen'
unless
defined?
(
Gitlab
::
Popen
)
module
Quality
class
KubernetesClient
attr_reader
:namespace
def
initialize
(
namespace:
ENV
[
'KUBE_NAMESPACE'
])
@namespace
=
namespace
end
def
cleanup
(
release_name
:)
command
=
[
'kubectl'
]
command
<<
%(-n "#{namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1)
command
<<
'|'
<<
%(grep "#{release_name}")
command
<<
'|'
<<
"awk '{print $1}'"
command
<<
'|'
<<
%(xargs kubectl -n "#{namespace}" delete)
command
<<
'||'
<<
'true'
run_command
(
command
)
end
private
def
run_command
(
command
)
puts
"Running command: `
#{
command
.
join
(
' '
)
}
`"
# rubocop:disable Rails/Output
Gitlab
::
Popen
.
popen_with_detail
(
command
)
end
end
end
This diff is collapsed.
Click to expand it.
scripts/review_apps/automated_cleanup.rb
0 → 100755
View file @
3299680c
# frozen_string_literal: true
require
'gitlab'
require_relative
File
.
expand_path
(
'../../lib/quality/helm_client.rb'
,
__dir__
)
require_relative
File
.
expand_path
(
'../../lib/quality/kubernetes_client.rb'
,
__dir__
)
class
AutomatedCleanup
attr_reader
:project_path
,
:gitlab_token
,
:cleaned_up_releases
def
initialize
(
project_path:
ENV
[
'CI_PROJECT_PATH'
],
gitlab_token:
ENV
[
'GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'
])
@project_path
=
project_path
@gitlab_token
=
gitlab_token
@cleaned_up_releases
=
[]
end
def
gitlab
@gitlab
||=
begin
Gitlab
.
configure
do
|
config
|
config
.
endpoint
=
'https://gitlab.com/api/v4'
# gitlab-bot's token "GitLab review apps cleanup"
config
.
private_token
=
gitlab_token
end
Gitlab
end
end
def
helm
@helm
||=
Quality
::
HelmClient
.
new
end
def
kubernetes
@kubernetes
||=
Quality
::
KubernetesClient
.
new
end
def
perform_gitlab_environment_cleanup!
(
days_for_stop
:,
days_for_delete
:)
puts
"Checking for review apps not updated in the last
#{
days_for_stop
}
days..."
checked_environments
=
[]
delete_threshold
=
threshold_time
(
days:
days_for_delete
)
stop_threshold
=
threshold_time
(
days:
days_for_stop
)
gitlab
.
deployments
(
project_path
,
per_page:
50
).
auto_paginate
do
|
deployment
|
next
unless
deployment
.
environment
.
name
.
start_with?
(
'review/'
)
next
if
checked_environments
.
include?
(
deployment
.
environment
.
slug
)
puts
checked_environments
<<
deployment
.
environment
.
slug
deployed_at
=
Time
.
parse
(
deployment
.
created_at
)
if
deployed_at
<
delete_threshold
print_release_state
(
subject:
'Review app'
,
release_name:
deployment
.
environment
.
slug
,
release_date:
deployment
.
created_at
,
action:
'deleting'
)
gitlab
.
delete_environment
(
project_path
,
deployment
.
environment
.
id
)
cleaned_up_releases
<<
deployment
.
environment
.
slug
elsif
deployed_at
<
stop_threshold
print_release_state
(
subject:
'Review app'
,
release_name:
deployment
.
environment
.
slug
,
release_date:
deployment
.
created_at
,
action:
'stopping'
)
gitlab
.
stop_environment
(
project_path
,
deployment
.
environment
.
id
)
cleaned_up_releases
<<
deployment
.
environment
.
slug
else
print_release_state
(
subject:
'Review app'
,
release_name:
deployment
.
environment
.
slug
,
release_date:
deployment
.
created_at
,
action:
'leaving'
)
end
end
end
def
perform_helm_releases_cleanup!
(
days
:)
puts
"Checking for Helm releases not updated in the last
#{
days
}
days..."
threshold_day
=
threshold_time
(
days:
days
)
helm
.
releases
(
args:
[
'--deployed'
,
'--failed'
,
'--date'
,
'--reverse'
,
'--max 25'
]).
each
do
|
release
|
next
if
cleaned_up_releases
.
include?
(
release
.
name
)
if
release
.
last_update
<
threshold_day
print_release_state
(
subject:
'Release'
,
release_name:
release
.
name
,
release_date:
release
.
last_update
,
action:
'cleaning'
)
helm
.
delete
(
release_name:
release
.
name
)
kubernetes
.
cleanup
(
release_name:
release
.
name
)
else
print_release_state
(
subject:
'Release'
,
release_name:
release
.
name
,
release_date:
release
.
last_update
,
action:
'leaving'
)
end
end
end
def
threshold_time
(
days
:)
Time
.
now
-
days
*
24
*
3600
end
def
print_release_state
(
subject
:,
release_name
:,
release_date
:,
action
:)
puts
"
\n
#{
subject
}
'
#{
release_name
}
' was last deployed on
#{
release_date
}
:
#{
action
}
it."
end
end
def
timed
(
task
)
start
=
Time
.
now
yield
(
self
)
puts
"
#{
task
}
finished in
#{
Time
.
now
-
start
}
seconds.
\n
"
end
automated_cleanup
=
AutomatedCleanup
.
new
timed
(
'Review apps cleanup'
)
do
automated_cleanup
.
perform_gitlab_environment_cleanup!
(
days_for_stop:
5
,
days_for_delete:
6
)
end
puts
timed
(
'Helm releases cleanup'
)
do
automated_cleanup
.
perform_helm_releases_cleanup!
(
days:
7
)
end
exit
(
0
)
This diff is collapsed.
Click to expand it.
scripts/review_apps/review-apps.sh
0 → 100755
View file @
3299680c
[[
"
$TRACE
"
]]
&&
set
-x
export
TILLER_NAMESPACE
=
"
$KUBE_NAMESPACE
"
function
check_kube_domain
()
{
if
[
-z
${
REVIEW_APPS_DOMAIN
+x
}
]
;
then
echo
"In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
echo
"You can do it in Auto DevOps project settings or defining a variable at group or project level"
echo
"You can also manually add it in .gitlab-ci.yml"
false
else
true
fi
}
function
download_gitlab_chart
()
{
curl
-o
gitlab.tar.bz2 https://gitlab.com/charts/gitlab/-/archive/
$GITLAB_HELM_CHART_REF
/gitlab-
$GITLAB_HELM_CHART_REF
.tar.bz2
tar
-xjf
gitlab.tar.bz2
cd
gitlab-
$GITLAB_HELM_CHART_REF
helm init
--client-only
helm repo add gitlab https://charts.gitlab.io
helm dependency update
helm dependency build
}
function
ensure_namespace
()
{
kubectl describe namespace
"
$KUBE_NAMESPACE
"
||
kubectl create namespace
"
$KUBE_NAMESPACE
"
}
function
install_tiller
()
{
echo
"Checking Tiller..."
helm init
--upgrade
kubectl rollout status
-n
"
$TILLER_NAMESPACE
"
-w
"deployment/tiller-deploy"
if
!
helm version
--debug
;
then
echo
"Failed to init Tiller."
return
1
fi
echo
""
}
function
create_secret
()
{
echo
"Create secret..."
kubectl create secret generic
-n
"
$KUBE_NAMESPACE
"
\
$CI_ENVIRONMENT_SLUG
-gitlab-initial-root-password
\
--from-literal
=
password
=
$REVIEW_APPS_ROOT_PASSWORD
\
--dry-run
-o
json | kubectl apply
-f
-
}
function
previousDeployFailed
()
{
set
+e
echo
"Checking for previous deployment of
$CI_ENVIRONMENT_SLUG
"
deployment_status
=
$(
helm status
$CI_ENVIRONMENT_SLUG
>
/dev/null 2>&1
)
status
=
$?
# if `status` is `0`, deployment exists, has a status
if
[
$status
-eq
0
]
;
then
echo
"Previous deployment found, checking status"
deployment_status
=
$(
helm status
$CI_ENVIRONMENT_SLUG
|
grep
^STATUS |
cut
-d
' '
-f2
)
echo
"Previous deployment state:
$deployment_status
"
if
[[
"
$deployment_status
"
==
"FAILED"
||
"
$deployment_status
"
==
"PENDING_UPGRADE"
||
"
$deployment_status
"
==
"PENDING_INSTALL"
]]
;
then
status
=
0
;
else
status
=
1
;
fi
else
echo
"Previous deployment NOT found."
fi
set
-e
return
$status
}
function
deploy
()
{
track
=
"
${
1
-stable
}
"
name
=
"
$CI_ENVIRONMENT_SLUG
"
if
[[
"
$track
"
!=
"stable"
]]
;
then
name
=
"
$name
-
$track
"
fi
replicas
=
"1"
service_enabled
=
"false"
postgres_enabled
=
"
$POSTGRES_ENABLED
"
gitlab_migrations_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
gitlab_sidekiq_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
gitlab_unicorn_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
gitlab_gitaly_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
gitlab_shell_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
gitlab_workhorse_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
if
[[
"
$CI_PROJECT_NAME
"
==
"gitlab-ee"
]]
;
then
gitlab_migrations_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
gitlab_sidekiq_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
gitlab_unicorn_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
gitlab_workhorse_image_repository
=
"registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
fi
# canary uses stable db
[[
"
$track
"
==
"canary"
]]
&&
postgres_enabled
=
"false"
env_track
=
$(
echo
$track
|
tr
-s
'[:lower:]'
'[:upper:]'
)
env_slug
=
$(
echo
${
CI_ENVIRONMENT_SLUG
//-/_
}
|
tr
-s
'[:lower:]'
'[:upper:]'
)
if
[[
"
$track
"
==
"stable"
]]
;
then
# for stable track get number of replicas from `PRODUCTION_REPLICAS`
eval
new_replicas
=
\$
${
env_slug
}
_REPLICAS
service_enabled
=
"true"
else
# for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
eval
new_replicas
=
\$
${
env_track
}
_
${
env_slug
}
_REPLICAS
fi
if
[[
-n
"
$new_replicas
"
]]
;
then
replicas
=
"
$new_replicas
"
fi
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
if
[
"
$CI_ENVIRONMENT_SLUG
"
!=
"production"
]
&&
previousDeployFailed
;
then
echo
"Deployment in bad state, cleaning up
$CI_ENVIRONMENT_SLUG
"
delete
cleanup
fi
helm repo add gitlab https://charts.gitlab.io/
helm dep update
.
HELM_CMD
=
$(
cat
<<
EOF
helm upgrade --install
\
--wait
\
--timeout 600
\
--set releaseOverride="
$CI_ENVIRONMENT_SLUG
"
\
--set global.hosts.hostSuffix="
$HOST_SUFFIX
"
\
--set global.hosts.domain="
$REVIEW_APPS_DOMAIN
"
\
--set certmanager.install=false
\
--set global.ingress.configureCertmanager=false
\
--set global.ingress.tls.secretName=tls-cert
\
--set global.ingress.annotations."external-dns
\.
alpha
\.
kubernetes
\.
io/ttl"="10"
--set gitlab.unicorn.resources.requests.cpu=200m
\
--set gitlab.sidekiq.resources.requests.cpu=100m
\
--set gitlab.gitlab-shell.resources.requests.cpu=100m
\
--set redis.resources.requests.cpu=100m
\
--set minio.resources.requests.cpu=100m
\
--set gitlab.migrations.image.repository="
$gitlab_migrations_image_repository
"
\
--set gitlab.migrations.image.tag="
$CI_COMMIT_REF_NAME
"
\
--set gitlab.sidekiq.image.repository="
$gitlab_sidekiq_image_repository
"
\
--set gitlab.sidekiq.image.tag="
$CI_COMMIT_REF_NAME
"
\
--set gitlab.unicorn.image.repository="
$gitlab_unicorn_image_repository
"
\
--set gitlab.unicorn.image.tag="
$CI_COMMIT_REF_NAME
"
\
--set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
\
--set gitlab.gitaly.image.tag="v
$GITALY_VERSION
"
\
--set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
\
--set gitlab.gitlab-shell.image.tag="v
$GITLAB_SHELL_VERSION
"
\
--set gitlab.unicorn.workhorse.image="
$gitlab_workhorse_image_repository
"
\
--set gitlab.unicorn.workhorse.tag="
$CI_COMMIT_REF_NAME
"
\
--namespace="
$KUBE_NAMESPACE
"
\
--version="
$CI_PIPELINE_ID
-
$CI_JOB_ID
"
\
"
$name
"
\
.
EOF
)
echo
"Deploying with:"
echo
$HELM_CMD
eval
$HELM_CMD
}
function
delete
()
{
track
=
"
${
1
-stable
}
"
name
=
"
$CI_ENVIRONMENT_SLUG
"
if
[[
"
$track
"
!=
"stable"
]]
;
then
name
=
"
$name
-
$track
"
fi
echo
"Deleting release '
$name
'..."
helm delete
--purge
"
$name
"
||
true
}
function
cleanup
()
{
echo
"Cleaning up
$CI_ENVIRONMENT_SLUG
..."
kubectl
-n
"
$KUBE_NAMESPACE
"
get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa 2>&1
\
|
grep
"
$CI_ENVIRONMENT_SLUG
"
\
|
awk
'{print $1}'
\
| xargs kubectl
-n
"
$KUBE_NAMESPACE
"
delete
\
||
true
}
This diff is collapsed.
Click to expand it.
spec/lib/quality/helm_client_spec.rb
0 → 100644
View file @
3299680c
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Quality
::
HelmClient
do
let
(
:namespace
)
{
'review-apps-ee'
}
let
(
:release_name
)
{
'my-release'
}
let
(
:raw_helm_list_result
)
do
<<~
OUTPUT
NAME REVISION UPDATED STATUS CHART NAMESPACE
review-improve-re-2dsd9d 1 Tue Jul 31 15:53:17 2018 FAILED gitlab-0.3.4
#{
namespace
}
review-11-1-stabl-3r2fso 1 Mon Jul 30 22:44:14 2018 FAILED gitlab-0.3.3
#{
namespace
}
review-49375-css-fk664j 1 Thu Jul 19 11:01:30 2018 FAILED gitlab-0.2.4
#{
namespace
}
OUTPUT
end
subject
{
described_class
.
new
(
namespace:
namespace
)
}
describe
'#releases'
do
it
'calls helm list with default arguments'
do
expect
(
Gitlab
::
Popen
).
to
receive
(
:popen_with_detail
)
.
with
([
%(helm list --namespace "#{namespace}")
])
.
and_return
(
Gitlab
::
Popen
::
Result
.
new
([],
''
))
subject
.
releases
end
it
'calls helm list with given arguments'
do
expect
(
Gitlab
::
Popen
).
to
receive
(
:popen_with_detail
)
.
with
([
%(helm list --namespace "#{namespace}" --deployed)
])
.
and_return
(
Gitlab
::
Popen
::
Result
.
new
([],
''
))
subject
.
releases
(
args:
[
'--deployed'
])
end
it
'returns a list of Release objects'
do
expect
(
Gitlab
::
Popen
).
to
receive
(
:popen_with_detail
)
.
with
([
%(helm list --namespace "#{namespace}" --deployed)
])
.
and_return
(
Gitlab
::
Popen
::
Result
.
new
([],
raw_helm_list_result
))
releases
=
subject
.
releases
(
args:
[
'--deployed'
])
expect
(
releases
.
size
).
to
eq
(
3
)
expect
(
releases
[
0
].
name
).
to
eq
(
'review-improve-re-2dsd9d'
)
expect
(
releases
[
0
].
revision
).
to
eq
(
1
)
expect
(
releases
[
0
].
last_update
).
to
eq
(
Time
.
parse
(
'Tue Jul 31 15:53:17 2018'
))
expect
(
releases
[
0
].
status
).
to
eq
(
'FAILED'
)
expect
(
releases
[
0
].
chart
).
to
eq
(
'gitlab-0.3.4'
)
expect
(
releases
[
0
].
namespace
).
to
eq
(
namespace
)
end
end
describe
'#delete'
do
it
'calls helm delete with default arguments'
do
expect
(
Gitlab
::
Popen
).
to
receive
(
:popen_with_detail
)
.
with
([
"helm delete --purge
#{
release_name
}
"
])
.
and_return
(
Gitlab
::
Popen
::
Result
.
new
([],
''
,
''
,
0
))
expect
(
subject
.
delete
(
release_name:
release_name
).
status
).
to
eq
(
0
)
end
end
end
This diff is collapsed.
Click to expand it.
spec/lib/quality/kubernetes_client_spec.rb
0 → 100644
View file @
3299680c
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Quality
::
KubernetesClient
do
subject
{
described_class
.
new
(
namespace:
'review-apps-ee'
)
}
describe
'#cleanup'
do
it
'calls kubectl with the correct arguments'
do
# popen_with_detail will receive an array with a bunch of arguments; we're
# only concerned with it having the correct namespace and release name
expect
(
Gitlab
::
Popen
).
to
receive
(
:popen_with_detail
)
do
|
args
|
expect
(
args
)
.
to
satisfy_one
{
|
arg
|
arg
.
start_with?
(
'-n "review-apps-ee" get'
)
}
expect
(
args
)
.
to
satisfy_one
{
|
arg
|
arg
==
'grep "my-release"'
}
expect
(
args
)
.
to
satisfy_one
{
|
arg
|
arg
.
end_with?
(
'-n "review-apps-ee" delete'
)
}
end
# We're not verifying the output here, just silencing it
expect
{
subject
.
cleanup
(
release_name:
'my-release'
)
}.
to
output
.
to_stdout
end
end
end
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment