Missing gem versions from index about rubygems.org HOT 8 CLOSED

require 'open-uri'
require 'zlib'
require 'progressbar'
require 'digest/md5'

$stdout.sync = true

# trap ctrl-c
trap('INT') do
  puts 'Exiting...'
  exit(1)
end

# make cache dir
Dir.mkdir('cache') unless Dir.exist?('cache')
# make output dir
Dir.mkdir('output') unless Dir.exist?('output')

def get_raw_specs
  # download full index from index.rubygem.org, and cache it to a file
  unless File.exist?('./cache/specs.4.8.gz')
    # write the index to a file
    File.open('./cache/specs.4.8.gz', 'wb') do |fo|
      fo.write URI.open('https://index.rubygems.org/specs.4.8.gz').read
    end
  end
  Marshal.load(Zlib::GzipReader.open('./cache/specs.4.8.gz').read)
end

def specs
  @specs ||= get_raw_specs.group_by { |name, _, _| name }
end

def cache_file_name(gem_name)
  "./cache/#{gem_name}-#{Digest::MD5.hexdigest(gem_name)}.info"
end

def get_raw_gem_info(name, force = false)
  fname = cache_file_name(name)
  # check if the gem info is cached
  if File.exist?(fname) && !force
    # read the cached gem info
    return File.open(fname, 'rb').read
  end


  # download the gem info from rubygems.org
  gem_info = URI.open("https://index.rubygems.org/info/#{name}").read
  # cache the gem info to a file
  File.open(fname, 'wb') do |fo|
    fo.write gem_info
  end
  gem_info
end

def check_for_missing_versions(gem_name)
  # get the list of versions from the index
  spec_versions = specs[gem_name].map do |_, version, platform|
    platform == 'ruby' ? version.to_s : "#{version.to_s}-#{platform}"
  end
  # get the list of versions from the gem info
  gem_info = get_raw_gem_info(gem_name)
  gem_info_versions = gem_info.split("\n")[1..].map{|row| row.split(" ")[0]}
  # check if the versions in the index are in the gem info
  spec_versions - gem_info_versions
end

def sillyize
  @silly = Thread.new do
    idx = 0
    chars = ["\u{15E1}","\u{15E7}","\u{2D54}"]
    while true
      @progress.format("%a %b#{chars[idx]}%i %p%% %t %R/sec")
      idx = (idx + 1) % 3
      sleep 0.1
    end
  end
end

def check_chunk(chunk, index)
  File.open("./output/missing_versions_#{index}.txt", 'w') do |fo|
    chunk.each do |name|
      begin
        missing_versions = check_for_missing_versions(name)
        unless missing_versions.empty?
          @progress.log "❌ #{name}: #{missing_versions.join(', ')}"
          fo.puts "#{name}: #{missing_versions.join(', ')}"
        end
        @progress.increment
      rescue => e
        puts "ERROR: #{e}"
        # abort the whole deal
        exit(1)
      end
    end
  end
end

def cache_chunk(chunk)
  chunk.each do |name|
    unless File.exist?(cache_file_name(name))
      get_raw_gem_info(name)
    end
    @progress.increment
  end
end

def cache_all
  @progress = ProgressBar.create(
    title: 'Caching',
    total: specs.keys.length,
    format: "%a %b\u{15E7}%i %p%% %t %R/sec",
    progress_mark: ' ',
    remainder_mark: "\u{FF65}",
    throttle_rate: 0.25
  )

  sillyize

  threads = []

  # split the keys into chunks of 1000
  specs.keys.each_slice(2500) do |chunk|
    threads << Thread.new { cache_chunk(chunk) }
  end
  threads.each(&:join)

  @silly.kill if @silly
  @silly = nil

  puts "done caching"
end

def check_all
  @progress = ProgressBar.create(
    title: 'Checking',
    total: specs.keys.length,
    format: "%a %b\u{15E7}%i %p%% %t %R/sec",
    progress_mark: ' ',
    remainder_mark: "\u{FF65}",
    throttle_rate: 0.25
  )

  sillyize

  idx = 0
  threads = []
  ops = []
  # split the keys into chunks of 1000
  specs.keys.each_slice(10000) do |chunk|
    ops << [chunk, idx]
    idx += 1
  end
  ops.each do |chunk, index|
    threads << Thread.new { check_chunk(chunk, index) }
  end
  threads.each(&:join)

  @silly.kill if @silly
  @silly = nil

  # concatentate the output files
  File.open('./output/missing_versions.txt', 'w') do |fo|
    Dir.glob('./output/missing_versions_*.txt').each do |file|
      fo.puts File.open(file, 'rb').read
    end
  end
end

cache_all
check_all