Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Boxiang Sun
gitlab-ce
Commits
093e9546
Commit
093e9546
authored
Aug 26, 2019
by
Peter Leitzen
Committed by
Dan Davison
Aug 26, 2019
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Extract gcloud and minikube cluster provider
Add cluster provider k3d
parent
94490b63
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
338 additions
and
88 deletions
+338
-88
qa/qa.rb
qa/qa.rb
+7
-0
qa/qa/service/cluster_provider/base.rb
qa/qa/service/cluster_provider/base.rb
+41
-0
qa/qa/service/cluster_provider/gcloud.rb
qa/qa/service/cluster_provider/gcloud.rb
+87
-0
qa/qa/service/cluster_provider/k3d.rb
qa/qa/service/cluster_provider/k3d.rb
+131
-0
qa/qa/service/cluster_provider/minikube.rb
qa/qa/service/cluster_provider/minikube.rb
+26
-0
qa/qa/service/kubernetes_cluster.rb
qa/qa/service/kubernetes_cluster.rb
+46
-88
No files found.
qa/qa.rb
View file @
093e9546
...
@@ -359,6 +359,13 @@ module QA
...
@@ -359,6 +359,13 @@ module QA
autoload
:KubernetesCluster
,
'qa/service/kubernetes_cluster'
autoload
:KubernetesCluster
,
'qa/service/kubernetes_cluster'
autoload
:Omnibus
,
'qa/service/omnibus'
autoload
:Omnibus
,
'qa/service/omnibus'
autoload
:Runner
,
'qa/service/runner'
autoload
:Runner
,
'qa/service/runner'
module
ClusterProvider
autoload
:Base
,
'qa/service/cluster_provider/base'
autoload
:Gcloud
,
'qa/service/cluster_provider/gcloud'
autoload
:Minikube
,
'qa/service/cluster_provider/minikube'
autoload
:K3d
,
'qa/service/cluster_provider/k3d'
end
end
end
##
##
...
...
qa/qa/service/cluster_provider/base.rb
0 → 100644
View file @
093e9546
# frozen_string_literal: true
module
QA
module
Service
module
ClusterProvider
class
Base
include
Service
::
Shellout
attr_reader
:rbac
def
initialize
(
rbac
:)
@rbac
=
rbac
end
def
cluster_name
@cluster_name
||=
"qa-cluster-
#{
Time
.
now
.
utc
.
strftime
(
"%Y%m%d%H%M%S"
)
}
-
#{
SecureRandom
.
hex
(
4
)
}
"
end
def
set_credentials
(
admin_user
)
raise
NotImplementedError
end
def
validate_dependencies
raise
NotImplementedError
end
def
setup
raise
NotImplementedError
end
def
teardown
raise
NotImplementedError
end
def
filter_credentials
(
credentials
)
credentials
end
end
end
end
end
qa/qa/service/cluster_provider/gcloud.rb
0 → 100644
View file @
093e9546
# frozen_string_literal: true
module
QA
module
Service
module
ClusterProvider
class
Gcloud
<
Base
def
validate_dependencies
find_executable
(
'gcloud'
)
||
raise
(
"You must first install `gcloud` executable to run these tests."
)
end
def
set_credentials
(
admin_user
)
master_auth
=
JSON
.
parse
(
`gcloud container clusters describe
#{
cluster_name
}
--region
#{
Runtime
::
Env
.
gcloud_region
}
--format 'json(masterAuth.username, masterAuth.password)'`
)
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
kubectl config set-credentials
#{
admin_user
}
--username
#{
master_auth
[
'masterAuth'
][
'username'
]
}
--password
#{
master_auth
[
'masterAuth'
][
'password'
]
}
CMD
end
def
setup
login_if_not_already_logged_in
create_cluster
end
def
teardown
delete_cluster
end
private
def
login_if_not_already_logged_in
if
Runtime
::
Env
.
has_gcloud_credentials?
attempt_login_with_env_vars
else
account
=
`gcloud auth list --filter=status:ACTIVE --format="value(account)"`
if
account
.
empty?
raise
"Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
puts
"gcloud account found. Using:
#{
account
}
for creating K8s cluster."
end
end
end
def
attempt_login_with_env_vars
puts
"No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
gcloud_account_key
=
Tempfile
.
new
(
'gcloud-account-key'
)
gcloud_account_key
.
write
(
Runtime
::
Env
.
gcloud_account_key
)
gcloud_account_key
.
close
gcloud_account_email
=
Runtime
::
Env
.
gcloud_account_email
shell
(
"gcloud auth activate-service-account
#{
gcloud_account_email
}
--key-file
#{
gcloud_account_key
.
path
}
"
)
ensure
gcloud_account_key
&&
gcloud_account_key
.
unlink
end
def
auth_options
"--enable-legacy-authorization"
unless
rbac
end
def
create_cluster
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
gcloud container clusters
create
#{
cluster_name
}
#{
auth_options
}
--enable-basic-auth
--region
#{
Runtime
::
Env
.
gcloud_region
}
--disk-size 10GB
--num-nodes
#{
Runtime
::
Env
.
gcloud_num_nodes
}
&& gcloud container clusters
get-credentials
--region
#{
Runtime
::
Env
.
gcloud_region
}
#{
cluster_name
}
CMD
end
def
delete_cluster
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
gcloud container clusters delete
--region
#{
Runtime
::
Env
.
gcloud_region
}
#{
cluster_name
}
--quiet --async
CMD
end
end
end
end
end
qa/qa/service/cluster_provider/k3d.rb
0 → 100644
View file @
093e9546
# frozen_string_literal: true
module
QA
module
Service
module
ClusterProvider
class
K3d
<
Base
def
validate_dependencies
find_executable
(
'k3d'
)
||
raise
(
"You must first install `k3d` executable to run these tests."
)
end
def
set_credentials
(
admin_user
)
end
def
setup
shell
"k3d create --workers 1 --name
#{
cluster_name
}
--wait 0"
@old_kubeconfig
=
ENV
[
'KUBECONFIG'
]
ENV
[
'KUBECONFIG'
]
=
fetch_kubeconfig
raise
"Could not fetch kubeconfig"
unless
ENV
[
'KUBECONFIG'
]
install_local_storage
end
def
teardown
ENV
[
'KUBECONFIG'
]
=
@old_kubeconfig
shell
"k3d delete --name
#{
cluster_name
}
"
end
# Fetch "real" certificate
# See https://github.com/rancher/k3s/issues/27
def
filter_credentials
(
credentials
)
kubeconfig
=
YAML
.
load_file
(
ENV
[
'KUBECONFIG'
])
ca_certificate
=
kubeconfig
.
dig
(
'clusters'
,
0
,
'cluster'
,
'certificate-authority-data'
)
credentials
.
merge
(
'data'
=>
credentials
[
'data'
].
merge
(
'ca.crt'
=>
ca_certificate
))
end
private
def
retry_until
(
max_attempts:
10
,
wait:
1
)
max_attempts
.
times
do
result
=
yield
return
result
if
result
sleep
wait
end
raise
"Retried
#{
max_attempts
}
times. Aborting"
end
def
fetch_kubeconfig
retry_until
do
config
=
`k3d get-kubeconfig --name
#{
cluster_name
}
`
.
chomp
config
if
config
=~
/kubeconfig.yaml/
end
end
def
install_local_storage
shell
(
'kubectl apply -f -'
,
stdin_data:
local_storage_config
)
end
# See https://github.com/rancher/k3d/issues/67
def
local_storage_config
<<~
YAML
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: storage-provisioner
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: storage-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:persistent-volume-provisioner
subjects:
- kind: ServiceAccount
name: storage-provisioner
namespace: kube-system
---
apiVersion: v1
kind: Pod
metadata:
name: storage-provisioner
namespace: kube-system
spec:
serviceAccountName: storage-provisioner
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
hostNetwork: true
containers:
- name: storage-provisioner
image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1
command: ["/storage-provisioner"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /tmp
name: tmp
volumes:
- name: tmp
hostPath:
path: /tmp
type: Directory
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: standard
namespace: kube-system
annotations:
storageclass.kubernetes.io/is-default-class: "true"
labels:
addonmanager.kubernetes.io/mode: EnsureExists
provisioner: k8s.io/minikube-hostpath
YAML
end
end
end
end
end
qa/qa/service/cluster_provider/minikube.rb
0 → 100644
View file @
093e9546
# frozen_string_literal: true
module
QA
module
Service
module
ClusterProvider
class
Minikube
<
Base
def
validate_dependencies
find_executable
(
'minikube'
)
||
raise
(
"You must first install `minikube` executable to run these tests."
)
end
def
set_credentials
(
admin_user
)
end
def
setup
shell
'minikube stop'
shell
"minikube profile
#{
cluster_name
}
"
shell
'minikube start'
end
def
teardown
shell
'minikube delete'
end
end
end
end
end
qa/qa/service/kubernetes_cluster.rb
View file @
093e9546
...
@@ -9,90 +9,63 @@ module QA
...
@@ -9,90 +9,63 @@ module QA
class
KubernetesCluster
class
KubernetesCluster
include
Service
::
Shellout
include
Service
::
Shellout
attr_reader
:api_url
,
:ca_certificate
,
:token
,
:rbac
attr_reader
:api_url
,
:ca_certificate
,
:token
,
:rbac
,
:provider
def
initialize
(
rbac:
true
)
def
initialize
(
rbac:
true
,
provider_class:
QA
::
Service
::
ClusterProvider
::
Gcloud
)
@rbac
=
rbac
@rbac
=
rbac
end
@provider
=
provider_class
.
new
(
rbac:
rbac
)
def
cluster_name
@cluster_name
||=
"qa-cluster-
#{
SecureRandom
.
hex
(
4
)
}
-
#{
Time
.
now
.
utc
.
strftime
(
"%Y%m%d%H%M%S"
)
}
"
end
end
def
create!
def
create!
validate_dependencies
validate_dependencies
login_if_not_already_logged_in
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
gcloud container clusters
create
#{
cluster_name
}
#{
auth_options
}
--enable-basic-auth
--region
#{
Runtime
::
Env
.
gcloud_region
}
--disk-size 10GB
--num-nodes
#{
Runtime
::
Env
.
gcloud_num_nodes
}
&& gcloud container clusters
get-credentials
--region
#{
Runtime
::
Env
.
gcloud_region
}
#{
cluster_name
}
CMD
@api_url
=
`kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}'`
@admin_user
=
"
#{
cluster_name
}
-admin"
master_auth
=
JSON
.
parse
(
`gcloud container clusters describe
#{
cluster_name
}
--region
#{
Runtime
::
Env
.
gcloud_region
}
--format 'json(masterAuth.username, masterAuth.password)'`
)
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
kubectl config set-credentials
#{
@admin_user
}
--username
#{
master_auth
[
'masterAuth'
][
'username'
]
}
--password
#{
master_auth
[
'masterAuth'
][
'password'
]
}
CMD
if
rbac
create_service_account
secrets
=
JSON
.
parse
(
`kubectl get secrets -o json`
)
@provider
.
validate_dependencies
gitlab_account
=
secrets
[
'items'
].
find
do
|
item
|
@provider
.
setup
item
[
'metadata'
][
'annotations'
][
'kubernetes.io/service-account.name'
]
==
'gitlab-account'
end
@ca_certificate
=
Base64
.
decode64
(
gitlab_account
[
'data'
][
'ca.crt'
])
@api_url
=
fetch_api_url
@token
=
Base64
.
decode64
(
gitlab_account
[
'data'
][
'token'
])
else
credentials
=
@provider
.
filter_credentials
(
fetch_credentials
)
@ca_certificate
=
Base64
.
decode64
(
`kubectl get secrets -o jsonpath="{.items[0].data['ca
\\
.crt']}"`
)
@ca_certificate
=
Base64
.
decode64
(
credentials
.
dig
(
'data'
,
'ca.crt'
))
@token
=
Base64
.
decode64
(
`kubectl get secrets -o jsonpath='{.items[0].data.token}'`
)
@token
=
Base64
.
decode64
(
credentials
.
dig
(
'data'
,
'token'
))
end
self
self
end
end
def
remove!
def
remove!
shell
<<~
CMD
.
tr
(
"
\n
"
,
' '
)
@provider
.
teardown
gcloud container clusters delete
end
--region
#{
Runtime
::
Env
.
gcloud_region
}
#{
cluster_name
}
def
cluster_name
--quiet --async
@provider
.
cluster_name
CMD
end
end
private
private
def
create_service_account
def
fetch_api_url
shell
(
'kubectl create -f -'
,
stdin_data:
service_account
)
`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'`
shell
(
"kubectl --user
#{
@admin_user
}
create -f -"
,
stdin_data:
service_account_role_binding
)
end
def
fetch_credentials
return
global_credentials
unless
rbac
@provider
.
set_credentials
(
admin_user
)
create_service_account
(
admin_user
)
account_credentials
end
end
def
service_account
def
admin_user
<<~
YAML
@admin_user
||=
"
#{
@provider
.
cluster_name
}
-admin"
end
def
create_service_account
(
user
)
service_account
=
<<~
YAML
---
apiVersion: v1
apiVersion: v1
kind: ServiceAccount
kind: ServiceAccount
metadata:
metadata:
name: gitlab-account
name: gitlab-account
namespace: default
namespace: default
YAML
---
end
def
service_account_role_binding
<<~
YAML
kind: ClusterRoleBinding
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
apiVersion: rbac.authorization.k8s.io/v1
metadata:
metadata:
...
@@ -106,39 +79,24 @@ module QA
...
@@ -106,39 +79,24 @@ module QA
name: cluster-admin
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
apiGroup: rbac.authorization.k8s.io
YAML
YAML
end
def
auth_options
shell
(
'kubectl apply -f -'
,
stdin_data:
service_account
)
"--enable-legacy-authorization"
unless
rbac
end
end
def
validate_dependencies
def
account_credentials
find_executable
(
'gcloud'
)
||
raise
(
"You must first install `gcloud` executable to run these tests."
)
secrets
=
JSON
.
parse
(
`kubectl get secrets -o json`
)
find_executable
(
'kubectl'
)
||
raise
(
"You must first install `kubectl` executable to run these tests."
)
end
def
login_if_not_already_logged_in
secrets
[
'items'
].
find
do
|
item
|
if
Runtime
::
Env
.
has_gcloud_credentials?
item
[
'metadata'
][
'annotations'
][
'kubernetes.io/service-account.name'
]
==
'gitlab-account'
attempt_login_with_env_vars
else
account
=
`gcloud auth list --filter=status:ACTIVE --format="value(account)"`
if
account
.
empty?
raise
"Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
puts
"gcloud account found. Using:
#{
account
}
for creating K8s cluster."
end
end
end
end
def
global_credentials
JSON
.
parse
(
`kubectl get secrets -o jsonpath='{.items[0]}'`
)
end
end
def
attempt_login_with_env_vars
def
validate_dependencies
puts
"No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
find_executable
(
'kubectl'
)
||
raise
(
"You must first install `kubectl` executable to run these tests."
)
gcloud_account_key
=
Tempfile
.
new
(
'gcloud-account-key'
)
gcloud_account_key
.
write
(
Runtime
::
Env
.
gcloud_account_key
)
gcloud_account_key
.
close
gcloud_account_email
=
Runtime
::
Env
.
gcloud_account_email
shell
(
"gcloud auth activate-service-account
#{
gcloud_account_email
}
--key-file
#{
gcloud_account_key
.
path
}
"
)
ensure
gcloud_account_key
&&
gcloud_account_key
.
unlink
end
end
end
end
end
end
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment