get_http_url (Raggle::Engine)

    # File raggle, line 5083
        def Engine::get_http_url(url, last_modified = nil)
          port = 80
          use_ssl = false
          user, pass = nil, nil

          # work with a copy of the url
          url = url.dup

          # check for ssl
          if url =~ /^https:/
            raise 'HTTPS support requires OpenSSL-Ruby' unless $HAVE_LIB['ssl']
            use_ssl, port = true, 443
          end

          # strip 'http://' prefix from URL
          url.gsub!(/^\w+?:\/\//, '') if url =~ /^\w+?:\/\//

          # get user and pass from URL
          if url =~ /^([^:]*):([^@]*)@/
            user, pass = $1, $2
            url.gsub!(/^.+?@/, '')
          end

          # get host and path portions of url
          raise "Couldn't parse URL: \"#{url}\"" unless url =~ /^(.+?)\/(.*)$/
          host, path = $1, $2

          # check for port in URL
          if host =~ /:(\d+)$/
            port = $1.to_i
            host.gsub!(/:(\d+)$/, '')
          end

          # start up proxy support
          proxy = Raggle::Proxy::find_proxy(host) || { }

          # initialize http connection
          http = Net::HTTP::new(host, port, proxy['host'], proxy['port'])

          # if we have SSL support and this is an https connection, then
          # enable SSL/TLS and disable peer verification (to turn off the
          # warning that's printed to the console)
          #
          # (Note: if we were doing this correctly, we'd have a proper CA
          # path)
          if $HAVE_LIB['ssl'] && use_ssl
            http.use_ssl = use_ssl
            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
          end

          # set http options
          http.open_timeout = $config['http_open_timeout'] || 30
          # http.read_timeout = $config['http_read_timeout'] || 120

          # start HTTP session
          http.start

          # raise an exception unless we could conenct
          raise "Couldn't connect to host \"#{host}:#{port}\"" unless http

          begin
            # check last-modified header first
            if last_modified
              # build headers
              headers = $config['use_http_headers'] ? $config['http_headers'] : {}
              headers['If-Modified-Since'] = last_modified

              # handle basic http authentication
              if user
                pass ||= ''
                headers['Authorization'] = 'Basic ' <<
                                           ["#{user}:#{pass}"].pack('m').strip
              end

              # get response
              resp = http.head('/' << path, headers)

              # check header
              if last_modified == resp['last-modified']
                # it's the same as the old content
                ret = [nil, last_modified]
              else
                # it's changed, get it again
                ret = get_http_url(url, nil)
              end
            else
              # no cache, just get the result
              headers = $config['use_http_headers'] ? $config['http_headers'] : {}

              # handle basic http authentication
              if user
                pass ||= ''
                headers['Authorization'] = 'Basic ' <<
                                           ["#{user}:#{pass}"].pack('m').strip
              end

              # get the result
              resp, content = http.get(('/' << path), headers)

              if resp.code !~ /2\d{2}/
                # if we got anything other than success, raise an exception
                # (this might be a little overzealous)
                raise "HTTP Error: #{resp.code} #{resp.message}"
              end

              # build return array
              ret = [content, resp['last-modified']]
            end
          rescue
            resp = $!.respond_to?(:response) ? $!.response : $!

            # handle unchanged content and redirects
            if resp.code.to_i == 304      # hasn't changed
              ret = [nil, last_modified]
            elsif resp.code =~ /3\d{2}/   # redirect
              location = resp['location']
              if location =~ /^\//
                # handle relative host, absolute path redirects
                location = host << ':' << port.to_s << location
                location = 'http' << (http.use_ssl ? 's' : '') << '://' << location
              elsif location !~ /^http/
                # handle relative host, relative path redirects
                path.gsub!(/[^\/]+$/, '') if path =~ /[^\/]+$/
                location = 'http' << (http.use_ssl ? 's' : '') << '://' <<
                           host << ':' << port.to_s << '/' << path << '/' <<
                           location
              end

              ret = Engine::get_http_url(location, last_modified)
            else
              raise "HTTP Error: #$!"
            end
          ensure
            # close HTTP connection
            # Note: if we don't specify this, then the connection is pooled
            # for the HTTP/1.1 spec (do we prefer that kind of behavior?
            # maybe I should make it an option)
            http.finish if http.active?
          end

          # return URL content and last-modified header
          ret
        end