Class: S3Browser::GemTasks

Inherits:
Rake::TaskLib
  • Object
show all
Includes:
Rake::DSL
Defined in:
lib/s3browser/gem_tasks.rb

Instance Method Summary collapse

Instance Method Details

#cli ⇒ Object 



143
144
145
 
# File 'lib/s3browser/gem_tasks.rb', line 143

def cli
  @cli ||= HighLine.new
end

#install_tasks ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/s3browser/gem_tasks.rb', line 10

def install_tasks
  require 'dotenv/tasks'

  namespace :s3browser do
    desc 'Run the web server for S3Browser'
    task :server do
      require 's3browser/server'
      app = S3Browser::Server
      app.set :environment, :production
      app.set :bind, '0.0.0.0'
      app.set :port, 9292
      app.run!
    end

    desc 'Fetch and store all the S3 objects'
    task :fetch do
      require 's3browser/fetch'
      S3Browser::Fetch.new.run
    end

    desc 'Set up the S3Browser'
    task :setup => :dotenv do
      require 'highline'
      require 'json'
      require 'aws-sdk'
      require 'time'
      require 'logger'
      # Aws.config.update(logger: Logger.new($stdout), log_level: :debug, log_formatter: Aws::Log::Formatter.colored)


      if File.exist?('.env') == false || cli.agree('Do you want to update your .env file (y/n)?') { |q| q.default = 'n' }
        setup_env
      end

      if cli.agree('Should we set up SQS notification on the S3 bucket for you (y/n)?') { |q| q.default = 'y' }
        setup_bucket
        setup_sqs
      end

      if cli.agree('Do you want to generate a Shoryuken worker config (y/n)?') { |q| q.default = 'y' }
        setup_worker_config
      end
    end
  end
end

#s3 ⇒ Object 



139
140
141
 
# File 'lib/s3browser/gem_tasks.rb', line 139

def s3
  @s3 ||= Aws::S3::Client.new
end

#setup_bucket ⇒ Object 



80
81
82
83
84
85
86
87
88
 
# File 'lib/s3browser/gem_tasks.rb', line 80

def setup_bucket
  # Ensure that the bucket exists
  s3.create_bucket({
    bucket: ENV['AWS_S3_BUCKET']
  })
  cli.say "Created the S3 bucket: #{ENV['AWS_S3_BUCKET']}"
rescue
  cli.say "Bucket already exists: #{ENV['AWS_S3_BUCKET']}"
end

#setup_env ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/s3browser/gem_tasks.rb', line 56

def setup_env
  envs = {}
  envs[:AWS_ACCESS_KEY_ID] = cli.ask('What is your AWS Access Key ID') { |q| q.validate = /^\w+$/; q.default = ENV['AWS_ACCESS_KEY_ID'] if ENV['AWS_ACCESS_KEY_ID'] }
  envs[:AWS_SECRET_ACCESS_KEY] = cli.ask('What is your AWS Secret Access Key') { |q| q.validate = /^[\w\/\+]+$/; q.default = ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY'] }
  envs[:AWS_REGION] = cli.ask('What AWS region should the service be located in') { |q| q.validate = /^[a-z]{2}\-[a-z]+\-\d$/; q.default = ENV['AWS_REGION'] if ENV['AWS_REGION'] }
  envs[:AWS_S3_BUCKET] = cli.ask('What is the name of the S3 bucket to use') { |q| q.validate = /^[^ ]+$/; ; q.default = ENV['AWS_S3_BUCKET'] if ENV['AWS_S3_BUCKET'] }
  envs[:AWS_SQS_QUEUE] = cli.ask('What is the name of the SQS queue to use') { |q| q.validate = /^[^ ]+$/; ; q.default = ENV['AWS_SQS_QUEUE'] if ENV['AWS_SQS_QUEUE'] }

  envs_string = envs.map {|k,v| "#{k}=#{v}"}.join("\n") + "\n"

  cli.say 'This is the proposed .env file:'
  cli.say envs_string + "\n"

  write_file = cli.agree 'Are you happy with the settings (y/n)? If yes, your current .env file will be overwritten'

  if write_file
    cli.say 'Writing .env file'
    File.open('.env', 'w') { |file| file.write('# export $(cat .env | grep -v ^# | xargs)' + "\n" + envs_string) }
    Dotenv.load! # Reload the .env files
  else
    cli.say 'Skipping the .env file'
  end
end

#setup_sqs ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/s3browser/gem_tasks.rb', line 90

def setup_sqs
  # Create the Queue
  resp = sqs.create_queue(
    queue_name: ENV['AWS_SQS_QUEUE']
  )
  queue_url = resp.to_h[:queue_url]
  cli.say "Created queue: #{queue_url}"

  # Replace the access policy on the queue to allow the bucket to write to it
  queue_arn = sqs.get_queue_attributes(queue_url: queue_url, attribute_names: ['QueueArn']).attributes['QueueArn']
  sqs.set_queue_attributes(
    queue_url: queue_url,
    attributes: {
      'Policy' => sqs_policy(ENV['AWS_S3_BUCKET'], queue_arn)
    }
  )
  cli.say 'Created the correct queue policy'

  # Ensure that the bucket pushes notifications to the queue
  s3.put_bucket_notification_configuration({
    bucket: ENV['AWS_S3_BUCKET'],
    notification_configuration: {
      queue_configurations: [
        {
          id: "S3BrowserNotification",
          queue_arn: queue_arn,
          events: ['s3:ObjectCreated:*','s3:ObjectRemoved:*']
        }
      ]
    }
  })
  cli.say 'Set the bucket to push notifications to SQS'
end

#setup_worker_config ⇒ Object 



124
125
126
127
128
129
130
131
132
133
 
# File 'lib/s3browser/gem_tasks.rb', line 124

def setup_worker_config
  config = {
    'concurrency': 1,
    'delay': 300,
    'queues': [ ENV['AWS_SQS_QUEUE'] ]
  }

  cli.say 'Writing shoryuken-config.yml'
  File.open('shoryuken-config.yml', 'w') { |file| file.write(config.to_yaml) }
end

#sqs ⇒ Object 



135
136
137
 
# File 'lib/s3browser/gem_tasks.rb', line 135

def sqs
  @sqs ||= Aws::SQS::Client.new
end

#sqs_policy(bucket_name, queue_arn) ⇒ Object 



147
148
149
150
151
152
153
154
 
# File 'lib/s3browser/gem_tasks.rb', line 147

def sqs_policy(bucket_name, queue_arn)
  folder = File.expand_path File.dirname(__FILE__)
  policy = JSON.parse(File.read("#{folder}/policy.json"))
  policy['Id'] = Time.now.strftime('%Y%m%dT%H%M%S')
  policy['Statement'][0]['Condition']['ArnLike']['aws:SourceArn'] = "arn:aws:s3:*:*:#{bucket_name}"
  policy['Statement'][0]['Resource'] = queue_arn
  JSON.generate(policy)
end