Clean up stale namespaces

scripts/review_apps/automated_cleanup.rb

@@ -116,6 +116,12 @@ class AutomatedCleanup
     delete_helm_releases(releases_to_delete)
   end
 
+  def perform_stale_namespace_cleanup!(days:)
+    kubernetes_client = Tooling::KubernetesClient.new(namespace: nil)
+
+    kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false)
+  end
+
   def perform_stale_pvc_cleanup!(days:)
     kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false)
   end
@@ -203,6 +209,10 @@ timed('Helm releases cleanup') do
   automated_cleanup.perform_helm_releases_cleanup!(days: 7)
 end
 
+timed('Stale Namespace cleanup') do
+  automated_cleanup.perform_stale_namespace_cleanup!(days: 14)
+end
+
 timed('Stale PVC cleanup') do
   automated_cleanup.perform_stale_pvc_cleanup!(days: 30)
 end

spec/tooling/lib/tooling/kubernetes_client_spec.rb

@@ -135,6 +135,52 @@ RSpec.describe Tooling::KubernetesClient do
     end
   end
 
+  describe '#cleanup_review_app_namespaces' do
+    let(:two_days_ago) { Time.now - 3600 * 24 * 2 }
+    let(:namespaces) { %w[review-abc-123 review-xyz-789] }
+
+    subject { described_class.new(namespace: nil) }
+
+    before do
+      allow(subject).to receive(:review_app_namespaces_created_before).with(created_before: two_days_ago).and_return(namespaces)
+    end
+
+    shared_examples 'a kubectl command to delete namespaces older than given creation time' do
+      let(:wait) { true }
+
+      specify do
+        expect(Gitlab::Popen).to receive(:popen_with_detail)
+                                   .with(["kubectl delete namespace " +
+                                            %(--now --ignore-not-found --wait=#{wait} #{namespaces.join(' ')})])
+                                   .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
+
+        # We're not verifying the output here, just silencing it
+        expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to output.to_stdout
+      end
+    end
+
+    it_behaves_like 'a kubectl command to delete namespaces older than given creation time'
+
+    it 'raises an error if the Kubernetes command fails' do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+                                 .with(["kubectl delete namespace " +
+                                          %(--now --ignore-not-found --wait=true #{namespaces.join(' ')})])
+                                 .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false)))
+
+      expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to raise_error(described_class::CommandFailedError)
+    end
+
+    context 'with no namespaces found' do
+      let(:namespaces) { [] }
+
+      it 'does not call #delete_namespaces_by_exact_names' do
+        expect(subject).not_to receive(:delete_namespaces_by_exact_names)
+
+        subject.cleanup_review_app_namespaces(created_before: two_days_ago)
+      end
+    end
+  end
+
   describe '#raw_resource_names' do
     it 'calls kubectl to retrieve the resource names' do
       expect(Gitlab::Popen).to receive(:popen_with_detail)
@@ -200,4 +246,49 @@ RSpec.describe Tooling::KubernetesClient do
       it_behaves_like 'a kubectl command to retrieve resource names sorted by creationTimestamp'
     end
   end
+
+  describe '#review_app_namespaces_created_before' do
+    let(:three_days_ago) { Time.now - 3600 * 24 * 3 }
+    let(:two_days_ago) { Time.now - 3600 * 24 * 2 }
+    let(:namespace_created_three_days_ago) { 'namespace-created-three-days-ago' }
+    let(:resource_type) { 'namespace' }
+    let(:raw_resources) do
+      {
+        items: [
+          {
+            apiVersion: "v1",
+            kind: "Namespace",
+            metadata: {
+              creationTimestamp: three_days_ago,
+              name: namespace_created_three_days_ago,
+              labels: {
+                tls: 'review-apps-tls'
+              }
+            }
+          },
+          {
+            apiVersion: "v1",
+            kind: "Namespace",
+            metadata: {
+              creationTimestamp: Time.now,
+              name: 'another-pvc',
+              labels: {
+                tls: 'review-apps-tls'
+              }
+            }
+          }
+        ]
+      }.to_json
+    end
+
+    specify do
+      expect(Gitlab::Popen).to receive(:popen_with_detail)
+                                 .with(["kubectl get namespace " \
+                                          "-l tls=review-apps-tls " \
+                                          "--sort-by='{.metadata.creationTimestamp}' -o json"])
+                                 .and_return(Gitlab::Popen::Result.new([], raw_resources, '', double(success?: true)))
+
+      expect(subject.__send__(:review_app_namespaces_created_before, created_before: two_days_ago)).to contain_exactly(namespace_created_three_days_ago)
+    end
+  end
 end

tooling/lib/tooling/kubernetes_client.rb

@@ -27,6 +27,13 @@ module Tooling
       delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
     end
 
+    def cleanup_review_app_namespaces(created_before:, wait: true)
+      namespaces = review_app_namespaces_created_before(created_before: created_before)
+      return if namespaces.empty?
+
+      delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait)
+    end
+
     private
 
     def delete_by_selector(release_name:, wait:)
@@ -66,6 +73,19 @@ module Tooling
       run_command(command)
     end
 
+    def delete_namespaces_by_exact_names(resource_names:, wait:)
+      command = [
+        'delete',
+        'namespace',
+        '--now',
+        '--ignore-not-found',
+        %(--wait=#{wait}),
+        resource_names.join(' ')
+      ]
+
+      run_command(command)
+    end
+
     def delete_by_matching_name(release_name:)
       resource_names = raw_resource_names
       command = [
@@ -101,9 +121,32 @@ module Tooling
       ]
 
       response = run_command(command)
-      JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
-        .map { |resource| resource.dig('metadata', 'name') if Time.parse(resource.dig('metadata', 'creationTimestamp')) < created_before }
-        .compact
+
+      resources_created_before_date(response, created_before)
+    end
+
+    def review_app_namespaces_created_before(created_before:)
+      command = [
+        'get',
+        'namespace',
+        "-l tls=review-apps-tls", # Get only namespaces used for review-apps
+        "--sort-by='{.metadata.creationTimestamp}'",
+        '-o json'
+      ]
+
+      response = run_command(command)
+
+      resources_created_before_date(response, created_before)
+    end
+
+    def resources_created_before_date(response, date)
+      items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
+
+      items.filter_map do |item|
+        item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp'))
+
+        item.dig('metadata', 'name') if item_created_at < date
+      end
     rescue ::JSON::ParserError => ex
       puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
       []