Class: Ci::BulkDeleteExpiredJobArtifactsWorker

Inherits:
Object
  • Object
show all
Includes:
ApplicationWorker, Gitlab::LoopHelpers, LimitedCapacity::Worker
Defined in:
app/workers/ci/bulk_delete_expired_job_artifacts_worker.rb

Constant Summary collapse

BATCH_SIZE = 
100
LOOP_LIMIT = 
500
LOOP_TIMEOUT = 
5.minutes

Constants included from ApplicationWorker

ApplicationWorker::LOGGING_EXTRA_KEY, ApplicationWorker::SAFE_PUSH_BULK_LIMIT

Constants included from Gitlab::Loggable

Gitlab::Loggable::ANONYMOUS

Constants included from WorkerAttributes

WorkerAttributes::DEFAULT_CONCURRENCY_LIMIT_PERCENTAGE_BY_URGENCY, WorkerAttributes::DEFAULT_DATA_CONSISTENCY, WorkerAttributes::DEFAULT_DATA_CONSISTENCY_PER_DB, WorkerAttributes::DEFAULT_DEFER_DELAY, WorkerAttributes::LOAD_BALANCED_DATA_CONSISTENCIES, WorkerAttributes::NAMESPACE_WEIGHTS, WorkerAttributes::VALID_DATA_CONSISTENCIES, WorkerAttributes::VALID_RESOURCE_BOUNDARIES, WorkerAttributes::VALID_URGENCIES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Gitlab::LoopHelpers

#loop_until

Methods included from LimitedCapacity::Worker

#perform, #remove_failed_jobs, #report_prometheus_metrics

Methods included from Gitlab::Loggable

#build_structured_payload

Methods included from Gitlab::SidekiqVersioning::Worker

#job_version

Methods included from WorkerContext

#with_context

Class Method Details

.max_running_jobs_limit ⇒ Object 



17
18
19
20
21
22
23
 
# File 'app/workers/ci/bulk_delete_expired_job_artifacts_worker.rb', line 17

def self.max_running_jobs_limit
  if Feature.enabled?(:bulk_delete_job_artifacts_high_concurrency, :instance)
    10
  else
    5
  end
end

Instance Method Details

#max_running_jobs ⇒ Object 



59
60
61
 
# File 'app/workers/ci/bulk_delete_expired_job_artifacts_worker.rb', line 59

def max_running_jobs
  self.class.max_running_jobs_limit
end

#perform_work ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'app/workers/ci/bulk_delete_expired_job_artifacts_worker.rb', line 25

def perform_work
  @mod_bucket = Gitlab::Ci::Artifacts::BucketManager.claim_bucket
  log_extra_metadata_on_done(:mod_bucket, @mod_bucket)
  return unless @mod_bucket

  @bucket_claimed = true
  removed_artifacts_count = 0

  loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do
    artifacts = get_artifacts
    if artifacts.empty?
      log_extra_metadata_on_done(:artifacts_empty, true)
      break
    end

    service_response = destroy_batch(artifacts)
    removed_artifacts_count += service_response[:destroyed_artifacts_count]
  end

  log_extra_metadata_on_done(:destroyed_job_artifacts_count, removed_artifacts_count)

  Gitlab::Ci::Artifacts::BucketManager.release_bucket(@mod_bucket, max_buckets: max_running_jobs)
  log_extra_metadata_on_done(:mod_bucket_released, @mod_bucket)
end

#remaining_work_count ⇒ Object 



50
51
52
53
54
55
56
57
 
# File 'app/workers/ci/bulk_delete_expired_job_artifacts_worker.rb', line 50

def remaining_work_count
  # Don't re-enqueue if we couldn't claim a bucket - let the cron job handle it
  return 0 unless @bucket_claimed

  # Fix when FF clean-up, as this query times-out without the exists?
  # https://console.postgres.ai/gitlab/gitlab-production-ci/sessions/46187/commands/141080
  Ci::JobArtifact.expired_before(Time.current).non_trace.artifact_unlocked.exists? ? 999 : 0
end