Module: Throttle

Included in:
Database
Defined in:
lib/issue_db/utils/throttle.rb

Instance Method Summary collapse

Instance Method Details

#fetch_rate_limit ⇒ Object 



4
5
6
7
8
 
# File 'lib/issue_db/utils/throttle.rb', line 4

def fetch_rate_limit
  @rate_limit_all = Retryable.with_context(:default) do
    @client.get("rate_limit")
  end
end

#rate_limit_details(type) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/issue_db/utils/throttle.rb', line 15

def rate_limit_details(type)
  # fetch the provided rate limit type
  # rate_limit resulting structure: {:limit=>5000, :used=>15, :remaining=>4985, :reset=>1713897293}
  rate_limit = @rate_limit_all[:resources][type]

  # calculate the time the rate limit will reset
  resets_at = Time.at(rate_limit[:reset]).utc

  return {
    rate_limit: rate_limit,
    resets_at: resets_at,
  }
end

#update_rate_limit(type) ⇒ Object

Update the in-memory “cached” rate limit value for the given rate limit type



11
12
13
 
# File 'lib/issue_db/utils/throttle.rb', line 11

def update_rate_limit(type)
  @rate_limit_all[:resources][type][:remaining] -= 1
end

#wait_for_rate_limit!(type = :core) ⇒ Object

A helper method to check the client’s current rate limit status before making a request NOTE: This method will sleep for the remaining time until the rate limit resets if the rate limit is hit :param: type [Symbol] the type of rate limit to check (core, search, graphql, etc) - default: :core :return: nil (nothing) - this method will block until the rate limit is reset for the given type



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/issue_db/utils/throttle.rb', line 33

def wait_for_rate_limit!(type = :core)
  @log.debug("checking rate limit status for type: #{type}")
  # make a request to get the comprehensive rate limit status
  # note: checking the rate limit status does not count against the rate limit in any way
  fetch_rate_limit if @rate_limit_all.nil?

  details = rate_limit_details(type)
  rate_limit = details[:rate_limit]
  resets_at = details[:resets_at]

  @log.debug(
    "rate_limit remaining: #{rate_limit.remaining} - " \
    "used: #{rate_limit.used} - " \
    "resets_at: #{resets_at} - " \
    "current time: #{Time.now}"
  )

  # exit early if the rate limit is not hit (we have remaining requests)
  unless rate_limit.remaining.zero?
    update_rate_limit(type)
    return
  end

  # if we make it here, we (probably) have hit the rate limit
  # fetch the rate limit again if we are at zero or if the rate limit reset time is in the past
  fetch_rate_limit if rate_limit.remaining.zero? || rate_limit.remaining < 0 || resets_at < Time.now

  details = rate_limit_details(type)
  rate_limit = details[:rate_limit]
  resets_at = details[:resets_at]

  # exit early if the rate limit is not actually hit (we have remaining requests)
  unless rate_limit.remaining.zero?
    @log.debug("rate_limit not hit - remaining: #{rate_limit.remaining}")
    update_rate_limit(type)
    return
  end

  # calculate the sleep duration - ex: reset time - current time
  sleep_duration = resets_at - Time.now
  @log.debug("sleep_duration: #{sleep_duration}")
  sleep_duration = [sleep_duration, 0].max # ensure sleep duration is not negative
  sleep_duration_and_a_little_more = sleep_duration.ceil + 2 # sleep a little more than the rate limit reset time

  # log the sleep duration and begin the blocking sleep call
  @log.info("github rate_limit hit: sleeping for: #{sleep_duration_and_a_little_more} seconds")
  sleep(sleep_duration_and_a_little_more)

  @log.info("github rate_limit sleep complete - Time.now: #{Time.now}")
end