Purging referrer URLs concurrently

GOOD_REFERER_RE = %r{^http://(www\.)?eigenclass\.org/hiki\.rb\?[^?;]+$|
                     http://(www\.)?anarchaia\.org}x
BAD_REFERER_RE = %r{a9\.com/|google.*/search|mail\.google\.|
                    bloglines\.com|eigenclass\.org|
                    search\?q=cache|search\.msn\.|blogsearch\.google\.|
                    del\.icio\.us|search\?q=|
                    comp\.lang\.ruby
                    }x 
LINK_TEXT = %r{eigenclass\.org}

def check_referer(referer, count, ok_referer_hash, referer_info_cache)
  record_good = lambda do
    ok_referer_hash[referer] = count
    referer_info_cache[referer] = true
  end
  record_bad = lambda{ referer_info_cache[referer] = false }

  record_bad.call unless %r{http://}i =~ referer

  begin
    if referer_info_cache.has_key?(referer)
      puts "CACHE HIT: #{referer}"
      record_good.call if referer_info_cache[referer]
    elsif GOOD_REFERER_RE =~ referer && BAD_REFERER_RE !~ referer
      record_good.call
    elsif BAD_REFERER_RE =~ referer
      # ignore them
      puts "DISCARDING: #{referer}"
      record_bad.call
    else
      times = 0
      begin
        Timeout.timeout(10) do
          open(referer) do |is|
            # could use a more sophisticated test (e.g. content classification)
            if is.read(100000)[LINK_TEXT]
              record_good.call
            else
              record_bad.call
            end
          end
        end
      rescue Timeout::Error
        times += 1
        if times < 3 
          retry
        else
          raise
        end
      end
    end
  rescue Timeout::Error
    puts "GET #{referer} timed out."
    record_bad.call
  end
end

require 'fileutils'
require 'poolingexecutor'
require 'open-uri'
require 'timeout'

def purge_bad_referers(filename, good_referrers = {})
  referer_hash = eval(File.read(filename))
  ok_referer_hash = {}
  executor = PoolingExecutor.new do |handlers|
    15.times do 
      # we could as well register a plain Object and use it as a "token",
      # with the actual code in the executor.run { ... }  block
      handlers << lambda do |referer, count|
        check_referer(referer, count, ok_referer_hash, good_referrers)
      end
    end
  end
  referer_hash.each do |referer, count|
    executor.run do |handler|
      puts "Processing #{referer}"
      puts "Keeping #{referer}" if handler[referer, count]
    end
  end
  executor.wait_for_all
  File.open(filename, "w") do |f|
    f.puts "{"
    ok_referer_hash.keys.sort.each do |key|
      f.puts "#{key.inspect} => #{ok_referer_hash[key].inspect},"
    end
    f.print "}"
  end
end

good_referrers = {}
ARGV.each do |fname|
  FileUtils.cp(fname, "#{fname}.bak")
  puts
  puts "=" * 80
  puts fname
  puts "=" * 80
  purge_bad_referers(fname, good_referrers)
end