Class: Ossert::Fetch::GitHub

Inherits:
Object
  • Object
show all
Extended by:
Forwardable
Defined in:
lib/ossert/fetch/github.rb

Constant Summary collapse

MAX_ATTEMPTS = 
5

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(project) ⇒ GitHub

Returns a new instance of GitHub.

Raises:

  • (ArgumentError) 


12
13
14
15
16
17
18
19
20
21
 
# File 'lib/ossert/fetch/github.rb', line 12

def initialize(project)
  @client = ::Octokit::Client.new(access_token: ENV['GITHUB_TOKEN'])
  client.default_media_type = 'application/vnd.github.v3.star+json'
  client.auto_paginate = true

  @project = project
  raise ArgumentError unless (@repo_name = project.github_alias).present?
  @owner = @repo_name.split('/')[0]
  @requests_count = 0
end

Instance Attribute Details

#client ⇒ Object (readonly)

Returns the value of attribute client.



7
8
9
 
# File 'lib/ossert/fetch/github.rb', line 7

def client
  @client
end

#project ⇒ Object (readonly)

Returns the value of attribute project.



7
8
9
 
# File 'lib/ossert/fetch/github.rb', line 7

def project
  @project
end

Instance Method Details

#branches(&block) ⇒ Object 



86
87
88
 
# File 'lib/ossert/fetch/github.rb', line 86

def branches(&block)
  request(:branches, @repo_name, &block)
end

#commit(sha) ⇒ Object 



115
116
117
 
# File 'lib/ossert/fetch/github.rb', line 115

def commit(sha)
  client.commit(@repo_name, sha)
end

#commits(from, to, &block) ⇒ Object 



94
95
96
 
# File 'lib/ossert/fetch/github.rb', line 94

def commits(from, to, &block)
  request(:commits, @repo_name, since: from, until: to, &block)
end

#commits_since(date) ⇒ Object 



132
133
134
 
# File 'lib/ossert/fetch/github.rb', line 132

def commits_since(date)
  client.commits_since(@repo_name, date)
end

#contributors(&block) ⇒ Object 



70
71
72
 
# File 'lib/ossert/fetch/github.rb', line 70

def contributors(&block)
  request(:contributors, @repo_name, anon: true, &block)
end

#date_from_tag(sha) ⇒ Object 



125
126
127
128
129
130
 
# File 'lib/ossert/fetch/github.rb', line 125

def date_from_tag(sha)
  tag_info = tag_info(sha)
  return tag_info[:tagger][:date] if tag_info
  value = commit(sha)[:commit][:committer][:date]
  DateTime.new(*value.split('-').map(&:to_i)).to_i
end

#forkers(&block) ⇒ Object 



82
83
84
 
# File 'lib/ossert/fetch/github.rb', line 82

def forkers(&block)
  request(:forks, @repo_name, &block)
end

#generate_anonymous ⇒ Object

GitHub sometimes hides login, this is fallback



529
530
531
532
533
 
# File 'lib/ossert/fetch/github.rb', line 529

def generate_anonymous
  @anonymous_count ||= 0
  @anonymous_count += 1
  "anonymous_#{@anonymous_count}"
end

#issue2pull_url(html_url) ⇒ Object 



207
208
209
210
211
212
 
# File 'lib/ossert/fetch/github.rb', line 207

def issue2pull_url(html_url)
  html_url.gsub(
    %r{https://github.com/(#{@repo_name})/pull/(\d+)},
    'https://api.github.com/repos/\2/pulls/\3'
  )
end

#issues(&block) ⇒ Object 



48
49
50
 
# File 'lib/ossert/fetch/github.rb', line 48

def issues(&block)
  request(:issues, @repo_name, state: :all, &block)
end

#issues_comments(&block) ⇒ Object 



52
53
54
 
# File 'lib/ossert/fetch/github.rb', line 52

def issues_comments(&block)
  request(:issues_comments, @repo_name, &block)
end

#last_year_commits ⇒ Object 



98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/ossert/fetch/github.rb', line 98

def last_year_commits
  last_year_commits = []
  retry_count = 3
  while last_year_commits.blank? && retry_count.positive?
    last_year_commits = client.commit_activity_stats(@repo_name)
    if last_year_commits.blank?
      sleep(15 * retry_count)
      retry_count -= 1
    end
  end
  last_year_commits
end

#latest_release ⇒ Object 



136
137
138
 
# File 'lib/ossert/fetch/github.rb', line 136

def latest_release
  @latest_release ||= client.latest_release(@repo_name)
end

#process ⇒ Object 



391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
 
# File 'lib/ossert/fetch/github.rb', line 391

def process
  contributors do |c|
     = c.try(:[], :login).presence || generate_anonymous
    community.total.contributors << 
  end
  community.total.users_involved += community.total.contributors
  community.total.users_involved.uniq!

  # TODO: extract contributors and commits, quarter by quarter.
  #
  # => {:sha=>"d1a43d32e615b4a75117151b002266c560ce9061",
  # :commit=>
  #   {:author=>
  #     {:name=>"Yves Senn",
  #     :email=>"[email protected]",
  #     :date=>"2015-09-22T08:25:14Z"},
  #   :committer=>
  #     {:name=>"Yves Senn",
  #     :email=>"[email protected]",
  #     :date=>"2015-09-22T08:25:14Z"},
  #   :message=>
  #     "Merge pull request #21678 from ronakjangir47/array_to_formatted_s_docs\n\nAdded Examples in docs for int
  #     ernal behavior of Array#to_formatted_s [ci skip]",
  #   :tree=>
  #     {:sha=>"204811aa155645b461467dbd2238ac41c0fe8a30",
  #     :url=>
  #       "https://api.github.com/repos/rails/rails/git/trees/204811aa155645b461467dbd2238ac41c0fe8a30"},
  #   :url=>
  #     "https://api.github.com/repos/rails/rails/git/commits/d1a43d32e615b4a75117151b002266c560ce9061",
  #   :comment_count=>0},
  # :url=>
  #   "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061",
  # :html_url=>
  #   "https://github.com/rails/rails/commit/d1a43d32e615b4a75117151b002266c560ce9061",
  # :comments_url=>
  #   "https://api.github.com/repos/rails/rails/commits/d1a43d32e615b4a75117151b002266c560ce9061/comments",
  # :author=>
  #   {:login=>"senny",
  #    ...
  #    :type=>"User",
  #    :site_admin=>false},
  # :committer=>
  #   {:login=>"senny",
  #    ...
  #   :type=>"User",
  #   :site_admin=>false},
  # :parents=>
  #   [{:sha=>"2a7e8f54c66dbd65822f2a7135546a240426b631",
  #     :url=>
  #     "https://api.github.com/repos/rails/rails/commits/2a7e8f54c66dbd65822f2a7135546a240426b631",
  #     :html_url=>
  #     "https://github.com/rails/rails/commit/2a7e8f54c66dbd65822f2a7135546a240426b631"},
  #   {:sha=>"192d29f1c7ea16c506c09da2b854d1acdfbc8749",
  #     :url=>
  #     "https://api.github.com/repos/rails/rails/commits/192d29f1c7ea16c506c09da2b854d1acdfbc8749",
  #     :html_url=>
  #     "https://github.com/rails/rails/commit/192d29f1c7ea16c506c09da2b854d1acdfbc8749"}]}

  # process collaborators and commits. year by year, more info then ^^^^^
  # count = 0; collab = Set.new; fetcher.commits(14.months.ago.utc.iso8601, 13.month.ago.utc.iso8601) do |commit|
  #   count+=1
  #   collab << (commit[:author].try(:[],:login) || commit[:commit][:author][:name])
  # end

  process_issues

  process_pulls

  process_actual_prs_and_issues

  process_last_release_date

  process_commits

  sleep(1)

  process_top_contributors

  sleep(1)

  branches do |branch|
    # stale and total
    # by quarter ? date from commit -> [:commit][:committer][:date]
    # 1. save dates by commit sha.
    branch_updated_at = commit(branch[:commit][:sha])[:commit][:committer][:date]
    stale_threshold = Time.now.beginning_of_quarter

    # 2. date -> total by quarter
    #    date -> stale
    agility.total.branches << branch[:name]
    agility.total.stale_branches << branch[:name] if branch_updated_at < stale_threshold
    agility.quarters[branch_updated_at].branches << branch[:name]
  end

  stargazers do |stargazer|
     = stargazer[:user][:login].presence || generate_anonymous
    community.total.stargazers << 
    community.total.users_involved << 

    community.quarters[stargazer[:starred_at]].stargazers << 
    community.quarters[stargazer[:starred_at]].users_involved << 
  end

  watchers do |watcher|
     = watcher[:login].presence || generate_anonymous
    community.total.watchers << 
    community.total.users_involved << 
  end

  forkers do |forker|
    community.total.forks << forker[:owner][:login]
    community.total.users_involved << forker[:owner][:login]
    community.quarters[forker[:created_at]].forks << forker[:owner][:login]
    community.quarters[forker[:created_at]].users_involved << forker[:owner][:login]
  end
rescue Octokit::NotFound => e
  project.github_alias = NO_GITHUB_NAME
  puts "Github NotFound Error: #{e.inspect}"
rescue Octokit::InvalidRepository => e
  puts "Github InvalidRepository Error: #{e.inspect}"
end

#process_actual_prs_and_issues ⇒ Object 



194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/ossert/fetch/github.rb', line 194

def process_actual_prs_and_issues
  actual_prs = Set.new
  actual_issues = Set.new
  agility.quarters.each_sorted do |_quarter, data|
    data.pr_actual = actual_prs.to_a
    data.issues_actual = actual_issues.to_a

    closed = Set.new(data.pr_closed + data.issues_closed)
    actual_prs = Set.new(actual_prs + data.pr_open) - closed
    actual_issues = Set.new(actual_issues + data.issues_open) - closed
  end
end

#process_closed_issue(issue) ⇒ Object 



303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/ossert/fetch/github.rb', line 303

def process_closed_issue(issue)
  agility.total.issues_closed << issue[:url]
  # if issue is closed for now, it also was opened somewhen
  agility.quarters[issue[:created_at]].issues_open << issue[:url]
  agility.quarters[issue[:closed_at]].issues_closed << issue[:url] if issue[:closed_at]

  return unless issue[:closed_at].present?
  days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
  @issues_processed_in_days << days_to_close
  (agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
end

#process_closed_pull(pull) ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
 
# File 'lib/ossert/fetch/github.rb', line 219

def process_closed_pull(pull)
  agility.total.pr_closed << pull[:url]
  agility.quarters[pull[:created_at]].pr_open << pull[:url]
  agility.quarters[pull[:closed_at]].pr_closed << pull[:url] if pull[:closed_at]
  agility.quarters[pull[:merged_at]].pr_merged << pull[:url] if pull[:merged_at]

  return unless pull[:closed_at].present?
  days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
  @pulls_processed_in_days << days_to_close
  (agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
end

#process_commits ⇒ Object 



150
151
152
153
154
155
156
157
158
 
# File 'lib/ossert/fetch/github.rb', line 150

def process_commits
  last_year_commits.each do |week|
    current_count = agility.total.last_year_commits.to_i
    agility.total.last_year_commits = current_count + week['total']

    current_quarter_count = agility.quarters[week['week']].commits.to_i
    agility.quarters[week['week']].commits = current_quarter_count + week['total']
  end
end

#process_issues ⇒ Object 



322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/ossert/fetch/github.rb', line 322

def process_issues
  @issues_processed_in_days = []

  issues do |issue|
    next if issue.key? :pull_request
    case issue[:state]
    when 'open'
      process_open_issue(issue)
    when 'closed'
      process_closed_issue(issue)
    end

    if issue[:user][:login] == @owner
      agility.total.issues_owner << issue[:url]
    else
      agility.total.issues_non_owner << issue[:url]
    end

    agility.total.issues_total << issue[:url]
    agility.quarters[issue[:created_at]].issues_total << issue[:url]

    created_at = issue[:created_at].to_datetime.to_i
    if agility.total.first_issue_date.zero? || created_at < agility.total.first_issue_date
      agility.total.first_issue_date = created_at
    end

    if agility.total.last_issue_date.zero? || created_at > agility.total.last_issue_date
      agility.total.last_issue_date = created_at
    end

    process_users_from_issue(issue)
  end

  values = @issues_processed_in_days.to_a.sort
  agility.total.issues_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
  agility.total.issues_processed_in_median = if values.count.odd?
                                            values[values.count / 2]
                                          elsif values.count.zero?
                                            0
                                          else
                                            ((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
                                          end

  issues_comments do |issue_comment|
     = issue_comment[:user].try(:[], :login).presence || generate_anonymous
    issue_url = /\A(.*)#issuecomment.*\z/.match(issue_comment[:html_url])[1]
    if issue_url.include?('/pull/') # PR comments are stored as Issue comments. Sadness =(
      if community.total.contributors.include? 
        agility.total.pr_with_contrib_comments << issue2pull_url(issue_url)
      end

      community.total.users_commenting_pr << 
      community.quarters[issue_comment[:created_at]].users_commenting_pr << 
      community.total.users_involved << 
      community.quarters[issue_comment[:created_at]].users_involved << 
      next
    end

    if community.total.contributors.include? 
      agility.total.issues_with_contrib_comments << issue_url
    end

    community.total.users_commenting_issues << 
    community.quarters[issue_comment[:created_at]].users_commenting_issues << 
    community.total.users_involved << 
    community.quarters[issue_comment[:created_at]].users_involved << 
  end
end

#process_last_release_date ⇒ Object 



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/ossert/fetch/github.rb', line 160

def process_last_release_date
  latest_release_date = 0

  tags do |tag|
    tag_date = date_from_tag(tag[:commit][:sha])
    latest_release_date = [latest_release_date, tag_date].max

    agility.total.releases_total_gh << tag[:name]
    agility.quarters[tag_date].releases_total_gh << tag[:name]
  end

  return if latest_release_date.zero?

  agility.total.last_release_date = latest_release_date # wrong: last_release_commit[:commit][:committer][:date]
  agility.total.commits_count_since_last_release = commits_since(Time.at(latest_release_date).utc).length
end

#process_open_issue(issue) ⇒ Object 



298
299
300
301
 
# File 'lib/ossert/fetch/github.rb', line 298

def process_open_issue(issue)
  agility.total.issues_open << issue[:url]
  agility.quarters[issue[:created_at]].issues_open << issue[:url]
end

#process_open_pull(pull) ⇒ Object 



214
215
216
217
 
# File 'lib/ossert/fetch/github.rb', line 214

def process_open_pull(pull)
  agility.total.pr_open << pull[:url]
  agility.quarters[pull[:created_at]].pr_open << pull[:url]
end

#process_pulls ⇒ Object 



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
 
# File 'lib/ossert/fetch/github.rb', line 238

def process_pulls
  @pulls_processed_in_days = Set.new

  retry_call do
    pulls do |pull|
      case pull[:state]
      when 'open'
        process_open_pull(pull)
      when 'closed'
        process_closed_pull(pull)
      end

      if pull[:user][:login] == @owner
        agility.total.pr_owner << pull[:url]
      else
        agility.total.pr_non_owner << pull[:url]
      end

      agility.total.pr_total << pull[:url]
      agility.quarters[pull[:created_at]].pr_total << pull[:url]

      created_at = pull[:created_at].to_datetime.to_i
      if agility.total.first_pr_date.zero? || created_at < agility.total.first_pr_date
        agility.total.first_pr_date = created_at
      end

      if agility.total.last_pr_date.zero? || created_at > agility.total.last_pr_date
        agility.total.last_pr_date = created_at
      end

      process_users_from_pull(pull)
    end
  end

  values = @pulls_processed_in_days.to_a.sort
  agility.total.pr_processed_in_avg = values.count.positive? ? values.sum / values.count : 0
  agility.total.pr_processed_in_median = if values.count.odd?
                                        values[values.count / 2]
                                      elsif values.count.zero?
                                        0
                                      else
                                        ((values[values.count / 2 - 1] + values[values.count / 2]) / 2.0).to_i
                                      end


  retry_call do
    pulls_comments do |pull_comment|
       = pull_comment[:user].try(:[], :login).presence || generate_anonymous
      if community.total.contributors.include? 
        agility.total.pr_with_contrib_comments << pull_comment[:pull_request_url]
      end

      community.total.users_commenting_pr << 
      community.quarters[pull_comment[:created_at]].users_commenting_pr << 
      community.total.users_involved << 
      community.quarters[pull_comment[:created_at]].users_involved << 
    end
  end
end

#process_quarters_issues_and_prs_processing_days ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/ossert/fetch/github.rb', line 177

def process_quarters_issues_and_prs_processing_days
  issues do |issue|
    next if issue.key? :pull_request
    next unless issue[:state] == 'closed'
    next unless issue[:closed_at].present?
    days_to_close = (Date.parse(issue[:closed_at]) - Date.parse(issue[:created_at])).to_i + 1
    (agility.quarters[issue[:closed_at]].issues_processed_in_days ||= []) << days_to_close
  end

  pulls do |pull|
    next unless pull[:state] == 'closed'
    next unless pull[:closed_at].present?
    days_to_close = (Date.parse(pull[:closed_at]) - Date.parse(pull[:created_at])).to_i + 1
    (agility.quarters[pull[:closed_at]].pr_processed_in_days ||= []) << days_to_close
  end
end

#process_top_contributors ⇒ Object

Add class with processing types, e.g. top_contributors, commits and so on



142
143
144
145
146
147
148
 
# File 'lib/ossert/fetch/github.rb', line 142

def process_top_contributors
  @top_contributors = (top_contributors || []).map { |contrib_data| contrib_data[:author][:login] }
  @top_contributors.last(10).reverse.each do ||
    (meta[:top_10_contributors] ||= []) << "https://github.com/#{login}"
  end
  nil
end

#process_users_from_issue(issue) ⇒ Object 



315
316
317
318
319
320
 
# File 'lib/ossert/fetch/github.rb', line 315

def process_users_from_issue(issue)
  community.total.users_creating_issues << issue[:user][:login]
  community.quarters[issue[:created_at]].users_creating_issues << issue[:user][:login]
  community.total.users_involved << issue[:user][:login]
  community.quarters[issue[:created_at]].users_involved << issue[:user][:login]
end

#process_users_from_pull(pull) ⇒ Object 



231
232
233
234
235
236
 
# File 'lib/ossert/fetch/github.rb', line 231

def process_users_from_pull(pull)
  community.total.users_creating_pr << pull[:user][:login]
  community.quarters[pull[:created_at]].users_creating_pr << pull[:user][:login]
  community.total.users_involved << pull[:user][:login]
  community.quarters[pull[:created_at]].users_involved << pull[:user][:login]
end

#pulls(&block) ⇒ Object 



56
57
58
59
60
61
 
# File 'lib/ossert/fetch/github.rb', line 56

def pulls(&block)
  # fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
  # http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
  # fetch comments and link with PR by "pull_request_url"
  request(:pulls, @repo_name, state: :all, &block)
end

#pulls_comments(&block) ⇒ Object 



63
64
65
66
67
68
 
# File 'lib/ossert/fetch/github.rb', line 63

def pulls_comments(&block)
  # fetch pull requests, identify by "url", store: "assignee", "milestone", created_at/updated_at, "user"
  # http://octokit.github.io/octokit.rb/Octokit/Client/PullRequests.html#pull_requests_comments-instance_method
  # fetch comments and link with PR by "pull_request_url"
  request(:pulls_comments, @repo_name, &block)
end

#request(endpoint, *args) ⇒ Object

TODO: Add github search feature def find_repo(user)

first_found = client.search_repos(project.name, language: :ruby, user: user)[:items].first
first_found.try(:[], :full_name)

end



29
30
31
32
33
34
 
# File 'lib/ossert/fetch/github.rb', line 29

def request(endpoint, *args)
  first_response_data = client.paginate(url(endpoint, args.shift), *args) do |_, last_response|
    last_response.data.each { |data| yield data }
  end
  first_response_data.each { |data| yield data }
end

#retry_call ⇒ Object 



515
516
517
518
519
520
521
522
523
524
525
526
 
# File 'lib/ossert/fetch/github.rb', line 515

def retry_call
  attempt = 0
  begin
    yield
  rescue Octokit::InternalServerError => e
    attempt += 1
    raise if attempt > MAX_ATTEMPTS
    puts "Github Error: #{e.inspect}... retrying"
    sleep(attempt * 5.minutes)
    retry
  end
end

#stargazers(&block) ⇒ Object 



74
75
76
 
# File 'lib/ossert/fetch/github.rb', line 74

def stargazers(&block)
  request(:stargazers, @repo_name, &block)
end

#tag_info(sha) ⇒ Object 



119
120
121
122
123
 
# File 'lib/ossert/fetch/github.rb', line 119

def tag_info(sha)
  client.tag(@repo_name, sha)
rescue Octokit::NotFound
  false
end

#tags(&block) ⇒ Object 



90
91
92
 
# File 'lib/ossert/fetch/github.rb', line 90

def tags(&block)
  request(:tags, @repo_name, &block)
end

#top_contributors ⇒ Object 



111
112
113
 
# File 'lib/ossert/fetch/github.rb', line 111

def top_contributors
  client.contributors_stats(@repo_name, retry_timeout: 5, retry_wait: 5)
end

#url(endpoint, repo_name) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/ossert/fetch/github.rb', line 36

def url(endpoint, repo_name)
  path = case endpoint
         when /issues_comments/
           'issues/comments'
         when /pulls_comments/
           'pulls/comments'
         else
           endpoint
         end
  "#{Octokit::Repository.path repo_name}/#{path}"
end

#watchers(&block) ⇒ Object 



78
79
80
 
# File 'lib/ossert/fetch/github.rb', line 78

def watchers(&block)
  request(:subscribers, @repo_name, &block)
end