Skip to content

G
gitlab-ce
Commit 5e5bb88d authored by Jacob Vosmaer's avatar Jacob Vosmaer
Browse files
Options

Remove code related to standalone development 

Remove code and documentation related to the processes
we followed when Workhorse development happened in its
own project.
parent bfd7e162
Hide whitespace changes
Inline Side-by-side
Showing with 17 additions and 656 deletions
workhorse/.gitlab-ci.yml deleted 100644 → 0
View file @ bfd7e162
workflow:
rules: &workflow_rules
# For merge requests, create a pipeline.
- if: '$CI_MERGE_REQUEST_IID'
# For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
# For tags, create a pipeline.
- if: '$CI_COMMIT_TAG'
# For stable branches, create a pipeline.
- if: '$CI_COMMIT_BRANCH =~ /^[\d-]+-stable$/'
default:
image: golang:1.16
tags:
- gitlab-org
# Disable DIND for SAST because we need to execute a before_script in the gosec-sast job
variables:
SAST_DISABLE_DIND: "true"
verify:
script:
- make verify
changelog:
script:
- _support/check_changelog.sh
rules:
- if: '$CI_MERGE_REQUEST_IID'
.test:
services:
- name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
# Disable the hooks so we don't have to stub the GitLab API
command: ["/usr/bin/env", "GITALY_TESTING_NO_GIT_HOOKS=1", "/scripts/process-wrapper"]
alias: gitaly
variables:
GITALY_ADDRESS: "tcp://gitaly:8075"
script:
- go version
- apt-get update && apt-get -y install libimage-exiftool-perl
- make test
test using go 1.15:
extends: .test
image: golang:1.15
test using go 1.16:
extends: .test
image: golang:1.16
test:release:
rules:
- if: '$CI_COMMIT_TAG'
script:
- git describe --exact-match
include:
- template: Security/SAST.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml
gosec-sast:
before_script:
- apk add make
- make install
rules: *workflow_rules
gemnasium-dependency_scanning:
rules: *workflow_rules
secret_detection:
rules: *workflow_rules
code_navigation:
image: golang:latest
allow_failure: true
script:
- go get github.com/sourcegraph/lsif-go/cmd/lsif-go
- lsif-go
artifacts:
reports:
lsif: dump.lsif
workhorse/.gitlab/CODEOWNERS deleted 100644 → 0
View file @ bfd7e162
* @jacobvosmaer-gitlab @nick.thomas @nolith @patrickbajao
workhorse/Makefile
View file @ 5e5bb88d
......@@ -2,9 +2,6 @@ PREFIX=/usr/local
PKG := gitlab.com/gitlab-org/gitlab/workhorse
BUILD_DIR ?= $(CURDIR)
TARGET_DIR ?= $(BUILD_DIR)/_build
TARGET_SETUP := $(TARGET_DIR)/.ok
BIN_BUILD_DIR := $(TARGET_DIR)/bin
COVERAGE_DIR := $(TARGET_DIR)/cover
VERSION_STRING := $(shell git describe)
ifeq ($(strip $(VERSION_STRING)),)
VERSION_STRING := v$(shell cat VERSION)
......@@ -32,53 +29,40 @@ endef
.PHONY: all
all: clean-build $(EXE_ALL)
$(TARGET_SETUP):
$(call message,"Setting up target directory")
rm -rf "$(TARGET_DIR)"
mkdir -p "$(TARGET_DIR)"
touch "$(TARGET_SETUP)"
.PHONY: gitlab-resize-image
gitlab-resize-image: $(TARGET_SETUP)
gitlab-resize-image:
$(call message,Building $@)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
.PHONY: gitlab-zip-cat
gitlab-zip-cat: $(TARGET_SETUP)
gitlab-zip-cat:
$(call message,Building $@)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
.PHONY: gitlab-zip-metadata
gitlab-zip-metadata: $(TARGET_SETUP)
gitlab-zip-metadata:
$(call message,Building $@)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
.PHONY: gitlab-workhorse
gitlab-workhorse: $(TARGET_SETUP)
gitlab-workhorse:
$(call message,Building $@)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)
.PHONY: install
install: $(EXE_ALL)
install: $(EXE_ALL)
$(call message,$@)
mkdir -p $(DESTDIR)$(PREFIX)/bin/
cd $(BUILD_DIR) && $(INSTALL) $(EXE_ALL) $(DESTDIR)$(PREFIX)/bin/
.PHONY: test
test: $(TARGET_SETUP) prepare-tests
test: prepare-tests
$(call message,$@)
@go test -tags "$(BUILD_TAGS)" ./...
@echo SUCCESS
.PHONY: coverage
coverage: $(TARGET_SETUP) prepare-tests
$(call message,$@)
@go test -tags "$(BUILD_TAGS)" -cover -coverprofile=test.coverage ./...
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
.PHONY: clean
clean: clean-workhorse clean-build
clean: clean-workhorse clean-build
$(call message,$@)
rm -rf testdata/data testdata/scratch
......@@ -87,20 +71,6 @@ clean-workhorse:
$(call message,$@)
rm -f $(EXE_ALL)
.PHONY: check-version
check-version:
@test -n "$(VERSION)" || (echo "VERSION not set." ; exit 1)
.PHONY: tag
tag: check-version
$(call message,$@)
sh _support/tag.sh "$(VERSION)"
.PHONY: signed_tag
signed_tag: check-version
$(call message,$@)
TAG_OPTS=-s sh _support/tag.sh "$(VERSION)"
.PHONY: clean-build
clean-build:
$(call message,$@)
......@@ -121,18 +91,18 @@ testdata/scratch:
verify: lint vet detect-context detect-assert check-formatting staticcheck deps-check
.PHONY: lint
lint: $(TARGET_SETUP)
lint:
$(call message,Verify: $@)
go install golang.org/x/lint/golint
@_support/lint.sh ./...
.PHONY: vet
vet: $(TARGET_SETUP)
vet:
$(call message,Verify: $@)
@go vet ./...
.PHONY: detect-context
detect-context: $(TARGET_SETUP)
detect-context:
$(call message,Verify: $@)
_support/detect-context.sh
......@@ -142,7 +112,7 @@ detect-assert:
_support/detect-assert.sh
.PHONY: check-formatting
check-formatting: $(TARGET_SETUP) install-goimports
check-formatting: install-goimports
$(call message,Verify: $@)
@_support/fmt.sh check
......@@ -150,7 +120,7 @@ check-formatting: $(TARGET_SETUP) install-goimports
# Additionally, megacheck will not return failure exit codes unless explicitly told to via the
# `-simple.exit-non-zero` `-unused.exit-non-zero` and `-staticcheck.exit-non-zero` flags
.PHONY: staticcheck
staticcheck: $(TARGET_SETUP)
staticcheck:
$(call message,Verify: $@)
go install honnef.co/go/tools/cmd/staticcheck
@ $(GOBIN)/staticcheck -go $(MINIMUM_SUPPORTED_GO_VERSION) ./...
......@@ -158,12 +128,12 @@ staticcheck: $(TARGET_SETUP)
# In addition to fixing imports, goimports also formats your code in the same style as gofmt
# so it can be used as a replacement.
.PHONY: fmt
fmt: $(TARGET_SETUP) install-goimports
fmt: install-goimports
$(call message,$@)
@_support/fmt.sh
.PHONY: goimports
install-goimports: $(TARGET_SETUP)
install-goimports:
$(call message,$@)
go install golang.org/x/tools/cmd/goimports
......
workhorse/PROCESS.md
View file @ 5e5bb88d
......@@ -4,154 +4,14 @@
GitLab-Workhorse has the following maintainers:
- Patrick Bajao `@patrickbajao`
- Alessio Caiazza `@nolith`
- Nick Thomas `@nick.thomas`
- Jacob Vosmaer `@jacobvosmaer-gitlab`
- Alessio Caiazza `@nolith`
This list is defined at https://about.gitlab.com/team/.
## Changelog
GitLab-Workhorse keeps a changelog which is generated when a new release
is created. The changelog is generated from entries that are included on each
merge request. To generate an entry on your branch run:
`_support/changelog "Change descriptions"`.
After the merge request is created, the ID of the merge request needs to be set
in the generated file. If you already know the merge request ID, run:
`_support/changelog -m <ID> "Change descriptions"`.
Any new merge request must contain either a new entry or a justification in the
merge request description why no changelog entry is needed.
This authoritative source for this list is https://about.gitlab.com/team/.
## Merging and reviewing contributions
Contributions must be reviewed by at least one Workhorse maintainer.
The final merge must be performed by a maintainer.
## Releases
> Below we describe the legacy release process, from when Workhorse
> had its own repository. These instructions are still useful for
> security backports.
New versions of Workhorse can be released by one of the Workhorse
maintainers. The release process is:
- pick a release branch. For x.y.0, use `master`. For all other
versions (x.y.1, x.y.2 etc.) , use `x-y-stable`. Also see [below](#versioning)
- run `make tag VERSION=x.y.z"` or `make signed_tag VERSION=x.y.z` on the release branch. This will
compile the changelog, bump the VERSION file, and make a tag matching it.
- push the branch and the tag to gitlab.com
- the new version will only be deployed to `gitlab.com` if [`GITLAB_WORKHORSE_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) is updated accordingly;
if applicable, please remind the person who originally asked for a new release to make this change
(the MR should include a link back to the [version tag](https://gitlab.com/gitlab-org/gitlab-workhorse/-/tags) and a copy of the changelog)
- the person who updates GITLAB_WORKHORSE_VERSION should also run `scripts/update-workhorse` after commiting the new GITLAB_WORKHORSE_VERSION. If they forget they will be reminded by CI.
## Security releases
Workhorse is included in the packages we create for GitLab, and each version of
GitLab specifies the version of Workhorse it uses in the `GITLAB_WORKHORSE_VERSION`
file, so security fixes in Workhorse are tightly coupled to the [general security release](https://about.gitlab.com/handbook/engineering/workflow/#security-issues)
workflow, with some elaborations to account for the changes happening across two
repositories. In particular, the Workhorse maintainer takes responsibility for
creating new patch versions of Workhorse that can be used in the security
release.
As security fixes are backported three releases in addition to master, and
changes need to happen across two repositories, up to eight merge requests, and
four Workhorse releases, can be required to fix a security issue in Workhorse.
This is a lot of overhead, so in general, it is better to fix security issues
without changing Workhorse. Where changes **are** necessary, this section
documents the necessary steps.
If you're working on a security fix in Workhorse, you need two sets of merge
requests:
* The fix itself, in the `gitlab-org/security/gitlab-workhorse` repository
* A merge request to change the version of workhorse included in the GitLab
security release, in the `gitlab-org/security/gitlab` repository.
If the Workhorse maintainer isn't also a GitLab maintainer, reviews will need to
be split across several people. If changes to GitLab **code** are required in
addition to the change of Workhorse version, they both happen in the same merge
request.
Start by creating a single merge request targeting `master` in Workhorse. Ensure
you include a changelog! If code changes are needed in GitLab as well, create a
GitLab merge request targeting `master` at this point, but don't worry about the
`GITLAB_WORKHORSE_VERSION` file yet.
Once the changes have passed review, the Workhorse maintainer will determine the
new versions of Workhorse that will be needed, and communicate that to the
author. To do this, examine the `GITLAB_WORKHORSE_VERSION` file on each GitLab
stable branch; for instance, if the security release consisted of GitLab
versions `12.10.1`, `12.9.2`, `12.8.3`, and `12.7.4`, we would see the following:
```
gitlab$ git fetch security master 12-10-stable-ee 12-9-stable-ee 12-8-stable-ee 12-7-stable-ee`
gitlab$ git show refs/remotes/security/master:GITLAB_WORKHORSE_VERSION
8.30.1
gitlab$ git show refs/remotes/security/12-10-stable-ee:GITLAB_WORKHORSE_VERSION
8.30.1
gitlab$ git show refs/remotes/security/12-9-stable-ee:GITLAB_WORKHORSE_VERSION
8.25.2
gitlab$ git show refs/remotes/security/12-8-stable-ee:GITLAB_WORKHORSE_VERSION
8.21.2
gitlab$ git show refs/remotes/security/12-7-stable-ee:GITLAB_WORKHORSE_VERSION
8.21.2
```
In this example, there are three distinct Workhorse stable branches to be
concerned with, plus Workhorse master: `8-30-stable`, `8-25-stable`, and
`8-21-stable`, and we can predict that we are going to need to create Workhorse
releases `8.30.2`, `8.25.3`, and `8.21.3`.
The author needs to create a merge request targeting each Workhorse stable
branch, and verify that the fix works once backported. They also need to create
(or update, if they already exist) GitLab merge requests, setting the
`GITLAB_WORKHORSE_VERSION` file to the predicted workhorse version, and assign
all the MRs back to the appropriate maintainer(s). The pipeline for the GitLab
MRs will fail until the Workhorse releases have been tagged; you can use the
`=workhorse_branch_name` syntax in the `GITLAB_WORKHORSE_VERSION` file to verify
that the MRs interact as expected, if necessary.
Once all involved maintainers are happy with the overall change, the Workhorse
maintainer will merge each of the Workhorse MRs and generate new Workhorse
releases from the stable branches. The tags will be present on the `security`
mirror and `dev.gitlab.org` **only** at this point.
Once the Workhorse tags exist, the GitLab maintainer ensures that all the GitLab
MRs are green and assigns those MRs on to the release bot.
The release managers merge the GitLab MRs, tag GitLab releases that reference
the new Workhorse tags, and release them in the usual way.
Once the security release is done, the Workhorse maintainer is responsible for
syncing the changes to the `gitlab-org/gitlab-workhorse` repository. Push the
changes to `master`, the new tags, and all the changes to the stable branches.
This process is quite involved, very manual, and extremely error-prone; work is
ongoing on automating it.
## Versioning
Workhorse uses a variation of SemVer. We don't use "normal" SemVer
because we have to be able to integrate into GitLab stable branches.
A version has the format MAJOR.MINOR.PATCH.
- Major and minor releases are tagged on the `master` branch
- If the change is backwards compatible, increment the MINOR counter
- If the change breaks compatibility, increment MAJOR and set MINOR to `0`
- Patch release tags must be made on stable branches
- Only make a patch release when targeting a GitLab stable branch
This means that tags that end in `.0` (e.g. `8.5.0`) must always be on
the master branch, and tags that end in anthing other than `.0` (e.g.
`8.5.2`) must always be on a stable branch.
> The reason we do this is that SemVer suggests something like a
> refactoring constitutes a "patch release", while the GitLab stable
> branch quality standards do not allow for back-porting refactorings
> into a stable branch.
workhorse/_support/changelog deleted 100755 → 0
View file @ bfd7e162
#!/usr/bin/env ruby
#
# Generate a changelog entry file in the correct location.
#
# Automatically stages the file and amends the previous commit if the `--amend`
# argument is used.
#
# Stolen from gitlab-org/gitaly, lifted from gitlab-org/gitlab-ce
require 'optparse'
require 'yaml'
Options = Struct.new(
:amend,
:author,
:dry_run,
:force,
:merge_request,
:title,
:type
)
INVALID_TYPE = -1
class ChangelogOptionParser
Type = Struct.new(:name, :description)
TYPES = [
Type.new('added', 'New feature'),
Type.new('fixed', 'Bug fix'),
Type.new('changed', 'Feature change'),
Type.new('deprecated', 'New deprecation'),
Type.new('removed', 'Feature removal'),
Type.new('security', 'Security fix'),
Type.new('performance', 'Performance improvement'),
Type.new('other', 'Other')
].freeze
TYPES_OFFSET = 1
class << self
def parse(argv)
options = Options.new
parser = OptionParser.new do |opts|
opts.banner = "Usage: #{__FILE__} [options] [title]\n\n"
# Note: We do not provide a shorthand for this in order to match the `git
# commit` interface
opts.on('--amend', 'Amend the previous commit') do |value|
options.amend = value
end
opts.on('-f', '--force', 'Overwrite an existing entry') do |value|
options.force = value
end
opts.on('-m', '--merge-request [integer]', Integer, 'Merge request ID') do |value|
options.merge_request = value
end
opts.on('-n', '--dry-run', "Don't actually write anything, just print") do |value|
options.dry_run = value
end
opts.on('-u', '--git-username', 'Use Git user.name configuration as the author') do |value|
options.author = git_user_name if value
end
opts.on('-t', '--type [string]', String, "The category of the change, valid options are: #{TYPES.map(&:name).join(', ')}") do |value|
options.type = parse_type(value)
end
opts.on('-h', '--help', 'Print help message') do
$stdout.puts opts
exit
end
end
parser.parse!(argv)
# Title is everything that remains, but let's clean it up a bit
options.title = argv.join(' ').strip.squeeze(' ').tr("\r\n", '')
options
end
def read_type
read_type_message
type = TYPES[$stdin.getc.to_i - TYPES_OFFSET]
assert_valid_type!(type)
type.name
end
private
def parse_type(name)
type_found = TYPES.find do |type|
type.name == name
end
type_found ? type_found.name : INVALID_TYPE
end
def read_type_message
$stdout.puts "\n>> Please specify the index for the category of your change:"
TYPES.each_with_index do |type, index|
$stdout.puts "#{index + TYPES_OFFSET}. #{type.description}"
end
$stdout.print "\n?> "
end
def assert_valid_type!(type)
unless type
$stderr.puts "Invalid category index, please select an index between 1 and #{TYPES.length}"
exit 1
end
end
def git_user_name
%x{git config user.name}.strip
end
end
end
class ChangelogEntry
attr_reader :options
def initialize(options)
@options = options
assert_feature_branch!
assert_title!
assert_new_file!
# Read type from $stdin unless is already set
options.type ||= ChangelogOptionParser.read_type
assert_valid_type!
$stdout.puts "\e[32mcreate\e[0m #{file_path}"
$stdout.puts contents
unless options.dry_run
write
amend_commit if options.amend
end
end
private
def contents
yaml_content = YAML.dump(
'title' => title,
'merge_request' => options.merge_request,
'author' => options.author,
'type' => options.type
)
remove_trailing_whitespace(yaml_content)
end
def write
File.write(file_path, contents)
end
def amend_commit
%x{git add #{file_path}}
exec("git commit --amend")
end
def fail_with(message)
$stderr.puts "\e[31merror\e[0m #{message}"
exit 1
end
def assert_feature_branch!
return unless branch_name == 'master'
fail_with "Create a branch first!"
end
def assert_new_file!
return unless File.exist?(file_path)
return if options.force
fail_with "#{file_path} already exists! Use `--force` to overwrite."
end
def assert_title!
return if options.title.length > 0 || options.amend
fail_with "Provide a title for the changelog entry or use `--amend`" \
" to use the title from the previous commit."
end
def assert_valid_type!
return unless options.type && options.type == INVALID_TYPE
fail_with 'Invalid category given!'
end
def title
if options.title.empty?
last_commit_subject
else
options.title
end
end
def last_commit_subject
%x{git log --format="%s" -1}.strip
end
def file_path
File.join(
unreleased_path,
branch_name.gsub(/[^\w-]/, '-') << '.yml'
)
end
def unreleased_path
path = File.join('changelogs', 'unreleased')
path = File.join('ee', path) if ee?
path
end
def ee?
@ee ||= File.exist?(File.expand_path('../CHANGELOG-EE.md', __dir__))
end
def branch_name
@branch_name ||= %x{git symbolic-ref --short HEAD}.strip
end
def remove_trailing_whitespace(yaml_content)
yaml_content.gsub(/ +$/, '')
end
end
if $0 == __FILE__
options = ChangelogOptionParser.parse(ARGV)
ChangelogEntry.new(options)
end
# vim: ft=ruby
workhorse/_support/check_changelog.sh deleted 100755 → 0
View file @ bfd7e162
#!/bin/sh
set -e
# we skip the changelog check if the merge requet title ends with "NO CHANGELOG"
if echo "$CI_MERGE_REQUEST_TITLE" | grep -q ' NO CHANGELOG$'; then
echo "Changelog not needed"
exit 0
fi
target=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}
if git diff --name-only "origin/$target" | grep -q '^changelogs/' ; then
echo "Changelog included"
else
echo "Please add a changelog running '_support/changelog'"
echo "or disable this check adding 'NO CHANGELOG' at the end of the merge request title"
echo "/title $CI_MERGE_REQUEST_TITLE NO CHANGELOG"
exit 1
fi
workhorse/_support/generate_changelog deleted 100755 → 0
View file @ bfd7e162
#!/usr/bin/env ruby
# Generates the changelog from the yaml entries in changelogs/unreleased
#
# Lifted form gitlab-org/gitaly
require 'yaml'
require 'fileutils'
class ChangelogEntry
attr_reader :title, :merge_request, :type, :author
def initialize(file_path)
yaml = YAML.safe_load(File.read(file_path))
@title = yaml['title']
@merge_request = yaml['merge_request']
@type = yaml['type']
@author = yaml['author']
end
def to_s
str = ""
str << "- #{title}\n"
str << " https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/#{merge_request}\n"
str << " Contributed by #{author}\n" if author
str
end
end
ROOT_DIR = File.expand_path('../..', __FILE__)
UNRELEASED_ENTRIES = File.join(ROOT_DIR, 'changelogs', 'unreleased')
CHANGELOG_FILE = File.join(ROOT_DIR, 'CHANGELOG')
def main(version)
entries = []
Dir["#{UNRELEASED_ENTRIES}/*.yml"].each do |yml|
entries << ChangelogEntry.new(yml)
FileUtils.rm(yml)
end
sections = []
types = entries.map(&:type).uniq.sort
types.each do |type|
text = ''
text << "### #{type.capitalize}\n"
entries.each do |e|
next unless e.type == type
text << e.to_s
end
sections << text
end
sections << '- No changes.' if sections.empty?
new_version_entry = ["## v#{version}\n\n", sections.join("\n"), "\n"].join
current_changelog = File.read(CHANGELOG_FILE).lines
header = current_changelog.shift(2)
new_changelog = [header, new_version_entry, current_changelog.join]
File.write(CHANGELOG_FILE, new_changelog.join)
end
unless ARGV.count == 1
warn "Usage: #{$0} VERSION"
warn "Specify version as x.y.z"
abort
end
main(ARGV.first)
workhorse/_support/tag.sh deleted 100644 → 0
View file @ bfd7e162
set -e
main() {
version=$1
set_version
changelog
git commit VERSION -m "Update VERSION to $version"
tag_name="v${version}"
git tag $TAG_OPTS -m "Version ${version}" -a ${tag_name}
git show ${tag_name}
cat <<'EOF'
Remember to now push your tag, either to gitlab.com (for a
normal release) or dev.gitlab.org (for a security release).
EOF
}
set_version() {
if ! echo "${version}" | grep -q '^[0-9]\+\.[0-9]\+\.[0-9]\+$' ; then
echo "Invalid VERSION: ${version}"
exit 1
fi
if git tag --list | grep -q "^v${version}$" ; then
echo "Tag already exists for ${version}"
exit 1
fi
echo "$version" > VERSION
}
changelog() {
_support/generate_changelog "$version"
git commit CHANGELOG changelogs/unreleased --file - <<EOF
Update CHANGELOG for ${version}
[ci skip]
EOF
}
main "$@"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7