class Bundler::Fetcher

Handles all the fetching with the rubygems server

Constants

AUTH_ERRORS

Exceptions classes that should bypass retry attempts. If your password didn't work the first time, it's not going to the third time.

HTTP_ERRORS

Attributes

api_timeout[RW]
disable_endpoint[RW]
max_retries[RW]
redirect_limit[RW]

Public Class Methods

new(remote_uri) click to toggle source
# File lib/bundler/fetcher.rb, line 93
def initialize(remote_uri)
  @redirect_limit = 5  # How many redirects to allow in one request
  @api_timeout    = 10 # How long to wait for each API call
  @max_retries    = 3  # How many retries for the API call

  @remote_uri = Bundler::Source.mirror_for(remote_uri)
  @public_uri = @remote_uri.dup
  @public_uri.user, @public_uri.password = nil, nil # don't print these

  Socket.do_not_reverse_lookup = true
  connection # create persistent connection
end

Public Instance Methods

_use_api(reraise_auth_error = false) click to toggle source
# File lib/bundler/fetcher.rb, line 234
def _use_api(reraise_auth_error = false)
  return @use_api if defined?(@use_api)

  if @remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
    @use_api = false
  elsif fetch(dependency_api_uri)
    @use_api = true
  end
rescue AuthenticationRequiredError => e
  raise e if reraise_auth_error
  false
rescue HTTPError
  @use_api = false
end
connection() click to toggle source
# File lib/bundler/fetcher.rb, line 106
def connection
  @connection ||= begin
    needs_ssl = @remote_uri.scheme == "https" ||
      Bundler.settings[:ssl_verify_mode] ||
      Bundler.settings[:ssl_client_cert]
    raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)

    con = Net::HTTP::Persistent.new 'bundler', :ENV

    if @remote_uri.scheme == "https"
      con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
        OpenSSL::SSL::VERIFY_PEER)
      con.cert_store = bundler_cert_store
    end

    if Bundler.settings[:ssl_client_cert]
      pem = File.read(Bundler.settings[:ssl_client_cert])
      con.cert = OpenSSL::X509::Certificate.new(pem)
      con.key  = OpenSSL::PKey::RSA.new(pem)
    end

    con.read_timeout = @api_timeout
    con.override_headers["User-Agent"] = self.class.user_agent
    con
  end
end
download_gem_from_uri(spec, uri) click to toggle source
# File lib/bundler/fetcher.rb, line 54
def download_gem_from_uri(spec, uri)
  spec.fetch_platform

  download_path = Bundler.requires_sudo? ? Bundler.tmp(spec.full_name) : Bundler.rubygems.gem_dir
  gem_path = "#{Bundler.rubygems.gem_dir}/cache/#{spec.full_name}.gem"

  FileUtils.mkdir_p("#{download_path}/cache")
  Bundler.rubygems.download_gem(spec, uri, download_path)

  if Bundler.requires_sudo?
    Bundler.mkdir_p "#{Bundler.rubygems.gem_dir}/cache"
    Bundler.sudo "mv #{Bundler.tmp(spec.full_name)}/cache/#{spec.full_name}.gem #{gem_path}"
  end

  gem_path
end
fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = []) click to toggle source

fetch index

# File lib/bundler/fetcher.rb, line 202
def fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = [])
  query_list = gem_names - full_dependency_list

  # only display the message on the first run
  if Bundler.ui.debug?
    Bundler.ui.debug "Query List: #{query_list.inspect}"
  else
    Bundler.ui.info ".", false
  end

  return {@remote_uri => last_spec_list} if query_list.empty?

  remote_specs = Bundler::Retry.new("dependency api", AUTH_ERRORS).attempts do
    fetch_dependency_remote_specs(query_list)
  end

  spec_list, deps_list = remote_specs
  returned_gems = spec_list.map {|spec| spec.first }.uniq
  fetch_remote_specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
rescue HTTPError, MarshalError, GemspecError
  Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
  Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
  @use_api = false
  return nil
end
fetch_spec(spec) click to toggle source

fetch a gem specification

# File lib/bundler/fetcher.rb, line 138
def fetch_spec(spec)
  spec = spec - [nil, 'ruby', '']
  spec_file_name = "#{spec.join '-'}.gemspec"

  uri = URI.parse("#{@remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
  if uri.scheme == 'file'
    Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
  elsif cached_spec_path = gemspec_cached_path(spec_file_name)
    Bundler.load_gemspec(cached_spec_path)
  else
    Bundler.load_marshal Gem.inflate(fetch(uri))
  end
rescue MarshalError
  raise HTTPError, "Gemspec #{spec} contained invalid data.\n"          "Your network or your gem server is probably having issues right now."
end
gemspec_cached_path(spec_file_name) click to toggle source

cached gem specification path, if one exists

# File lib/bundler/fetcher.rb, line 156
def gemspec_cached_path spec_file_name
  paths = Bundler.rubygems.spec_cache_dirs.map { |dir| File.join(dir, spec_file_name) }
  paths = paths.select {|path| File.file? path }
  paths.first
end
inspect() click to toggle source
# File lib/bundler/fetcher.rb, line 249
def inspect
  "#<#{self.class}:0x#{object_id} uri=#{uri}>"
end
specs(gem_names, source) click to toggle source

return the specs in the bundler format as an index

# File lib/bundler/fetcher.rb, line 163
def specs(gem_names, source)
  old = Bundler.rubygems.sources
  index = Index.new

  if gem_names && use_api
    specs = fetch_remote_specs(gem_names)
  end

  if specs.nil?
    # API errors mean we should treat this as a non-API source
    @use_api = false

    specs = Bundler::Retry.new("source fetch", AUTH_ERRORS).attempts do
      fetch_all_remote_specs
    end
  end

  specs[@remote_uri].each do |name, version, platform, dependencies|
    next if name == 'bundler'
    spec = nil
    if dependencies
      spec = EndpointSpecification.new(name, version, platform, dependencies)
    else
      spec = RemoteSpecification.new(name, version, platform, self)
    end
    spec.source = source
    spec.source_uri = @remote_uri
    index << spec
  end

  index
rescue CertificateFailureError => e
  Bundler.ui.info "" if gem_names && use_api # newline after dots
  raise e
ensure
  Bundler.rubygems.sources = old
end
uri() click to toggle source
# File lib/bundler/fetcher.rb, line 133
def uri
  @public_uri
end
use_api() click to toggle source
# File lib/bundler/fetcher.rb, line 228
def use_api
  _use_api(true)
rescue AuthenticationRequiredError
  retry_with_auth{_use_api(false)}
end
user_agent() click to toggle source
# File lib/bundler/fetcher.rb, line 71
def user_agent
  @user_agent ||= begin
    ruby = Bundler.ruby_version

    agent = "bundler/#{Bundler::VERSION}"
    agent += " rubygems/#{Gem::VERSION}"
    agent += " ruby/#{ruby.version}"
    agent += " (#{ruby.host})"
    agent += " command/#{ARGV.first}"

    if ruby.engine != "ruby"
      # engine_version raises on unknown engines
      engine_version = ruby.engine_version rescue "???"
      agent += " #{ruby.engine}/#{engine_version}"
    end
    # add a random ID so we can consolidate runs server-side
    agent << " " << SecureRandom.hex(8)
  end
end

Private Instance Methods

bundler_cert_store() click to toggle source
# File lib/bundler/fetcher.rb, line 362
def bundler_cert_store
  store = OpenSSL::X509::Store.new
  if Bundler.settings[:ssl_ca_cert]
    if File.directory? Bundler.settings[:ssl_ca_cert]
      store.add_path Bundler.settings[:ssl_ca_cert]
    else
      store.add_file Bundler.settings[:ssl_ca_cert]
    end
  else
    store.set_default_paths
    certs = File.expand_path("../ssl_certs/*.pem", __FILE__)
    Dir.glob(certs).each { |c| store.add_file c }
  end
  store
end
dependency_api_uri(gem_names = []) click to toggle source
# File lib/bundler/fetcher.rb, line 303
def dependency_api_uri(gem_names = [])
  uri = fetch_uri + "api/v1/dependencies"
  uri.query = "gems=#{URI.encode(gem_names.join(","))}" if gem_names.any?
  uri
end
fetch(uri, counter = 0) click to toggle source
# File lib/bundler/fetcher.rb, line 262
def fetch(uri, counter = 0)
  raise HTTPError, "Too many redirects" if counter >= @redirect_limit

  response = request(uri)
  Bundler.ui.debug("HTTP #{response.code} #{response.message}")

  case response
  when Net::HTTPRedirection
    new_uri = URI.parse(response["location"])
    if new_uri.host == uri.host
      new_uri.user = uri.user
      new_uri.password = uri.password
    end
    fetch(new_uri, counter + 1)
  when Net::HTTPSuccess
    response.body
  when Net::HTTPRequestEntityTooLarge
    raise FallbackError, response.body
  when Net::HTTPUnauthorized
    raise AuthenticationRequiredError, "#{response.class}: #{response.body}"
  else
    raise HTTPError, "#{response.class}: #{response.body}"
  end
end
fetch_all_remote_specs() click to toggle source

fetch from modern index: specs.4.8.gz

# File lib/bundler/fetcher.rb, line 330
def fetch_all_remote_specs
  old_sources = Bundler.rubygems.sources
  Bundler.rubygems.sources = [@remote_uri.to_s]
  Bundler.rubygems.fetch_all_remote_specs
rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError => e
  case e.message
  when /certificate verify failed/
    raise CertificateFailureError.new(uri)
  when /401|403/
    # Gemfury uses a 403 for unauthenticated requests instead of a 401, so retry auth
    # on both.
    retry_with_auth { fetch_all_remote_specs }
  else
    Bundler.ui.trace e
    raise HTTPError, "Could not fetch specs from #{uri}"
  end
ensure
  Bundler.rubygems.sources = old_sources
end
fetch_dependency_remote_specs(gem_names) click to toggle source

fetch from Gemcutter Dependency Endpoint API

# File lib/bundler/fetcher.rb, line 310
def fetch_dependency_remote_specs(gem_names)
  Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(',')}"
  marshalled_deps = fetch dependency_api_uri(gem_names)
  gem_list = Bundler.load_marshal(marshalled_deps)
  deps_list = []

  spec_list = gem_list.map do |s|
    dependencies = s[:dependencies].map do |name, requirement|
      dep = well_formed_dependency(name, requirement.split(", "))
      deps_list << dep.name
      dep
    end

    [s[:name], Gem::Version.new(s[:number]), s[:platform], dependencies]
  end

  [spec_list, deps_list.uniq]
end
fetch_uri() click to toggle source
# File lib/bundler/fetcher.rb, line 395
def fetch_uri
  @fetch_uri ||= begin
    if @remote_uri.host == "rubygems.org"
      uri = @remote_uri.dup
      uri.host = "bundler.rubygems.org"
      uri
    else
      @remote_uri
    end
  end
end
request(uri) click to toggle source
# File lib/bundler/fetcher.rb, line 287
def request(uri)
  Bundler.ui.debug "HTTP GET #{uri}"
  req = Net::HTTP::Get.new uri.request_uri
  if uri.user
    user = CGI.unescape(uri.user)
    password = uri.password ? CGI.unescape(uri.password) : nil
    req.basic_auth(user, password)
  end
  connection.request(uri, req)
rescue OpenSSL::SSL::SSLError
  raise CertificateFailureError.new(uri)
rescue *HTTP_ERRORS => e
  Bundler.ui.trace e
  raise HTTPError, "Network error while fetching #{uri}"
end
retry_with_auth() { || ... } click to toggle source

Attempt to retry with HTTP authentication, if it's appropriate to do so. Yields to a block; the caller should use this to re-attempt the failing request with the altered `@remote_uri`.

# File lib/bundler/fetcher.rb, line 380
def retry_with_auth
  # Authentication has already been attempted and failed.
  raise BadAuthenticationError.new(uri) if @remote_uri.user

  auth = Bundler.settings[@remote_uri.to_s]

  # Authentication isn't provided at all, by "bundle config" or in the URI.
  raise AuthenticationRequiredError.new(uri) if auth.nil?

  @remote_uri.user, @remote_uri.password = *auth.split(":", 2)
  yield
end
well_formed_dependency(name, *requirements) click to toggle source
# File lib/bundler/fetcher.rb, line 350
def well_formed_dependency(name, *requirements)
  Gem::Dependency.new(name, *requirements)
rescue ArgumentError => e
  illformed = 'Ill-formed requirement ["#<YAML::Syck::DefaultKey'
  raise e unless e.message.include?(illformed)
  puts # we shouldn't print the error message on the "fetching info" status line
  raise GemspecError,
    "Unfortunately, the gem #{s[:name]} (#{s[:number]}) has an invalid "          "gemspec. \nPlease ask the gem author to yank the bad version to fix "          "this issue. For more information, see http://bit.ly/syck-defaultkey."
end