Hello, I am working on creating a proxy server based off the example given on the poe.perl.org website for a HTTP Proxy.
I have tried to change it to use streaming, however, I am finding that often the responses are incomplete and the wheels in POE::Component::Client::HTTP report read errors. This problem seems to be particularily bad when multiple requests are being made at the same time. However, when I do not use streaming, everything works fine. Interestingly, even without streaming there are still some read errors, but they do not seem to affect the content returned in the response object. I am making this proxy for a Windows system. I would be very happy if you could quickly take a look at the code to try and see why this is not working. The problem may be in the way POE::Component::Client::HTTP handles streaming (I am using v 1.56 2004/07/13 18:02:37 rcaputo), but more likely its the way I am interfacing with-it. The only routine changed heavily is handle_http_response from the example off the POE website. Thanks in advance, George Pabst [EMAIL PROTECTED] #!/usr/bin/perl use warnings; use strict; use POE; use POE::Component::Server::TCP; use POE::Component::Client::HTTP; use POE::Filter::HTTPD; use HTTP::Response; use Compress::Zlib; sub DUMP_REQUESTS () { 0 } sub DUMP_RESPONSES () { 0 } sub LISTEN_PORT () { 8088 } ### Spawn a web client to fetch requests through. our $HTTP_VER = '1.0'; # Version of HTTP to report to servers and clients our $COMPRESS_TEXT = 1; # GZIP compress HTML and text our $CRLF = "\015\012"; our $COMPRESS_TEXT = 0; POE::Component::Client::HTTP->spawn(Protocol => "HTTP/$HTTP_VER", Alias => 'ua', Agent => 'Mozilla/4.0 (compatible;)', Streaming => 4096, FollowRedirects => 0); ### Spawn a web server. # The ClientInput function is called to deal with client input. # ClientInput's callback function will receive entire HTTP requests # because this server uses POE::Filter::HTTPD to parse its input. # # InlineStates let us attach our own events and handlers to a TCP # server. Here we attach a handler for the got_response event, which # will be sent to us by Client::HTTP when it has fetched something. POE::Component::Server::TCP->new ( Alias => "web_server", Port => LISTEN_PORT, ClientFilter => 'POE::Filter::HTTPD', ClientInput => \&handle_http_request, InlineStates => { got_response => \&handle_http_response, }, ); ### Run the proxy until it is done, then exit. POE::Kernel->run(); exit 0; ### Handle HTTP requests from the client. Pass them to the HTTP ### client component for further processing. Optionally dump the ### request as text to STDOUT. sub handle_http_request { my ( $kernel, $heap, $request ) = @_[ KERNEL, HEAP, ARG0 ]; # If the request is really a HTTP::Response, then it indicates a # problem parsing the client's request. Send the response back so # the client knows what's happened. if ( $request->isa("HTTP::Response") ) { $heap->{client}->put($request); $kernel->yield("shutdown"); return; } # Client::HTTP doesn't support keep-alives yet. $request->header( "Connection", "close" ); $request->header( "Proxy-Connection", "close" ); $request->remove_header("Keep-Alive"); display_thing( $request->as_string() ) if DUMP_REQUESTS; $heap->{client}->set_output_filter(POE::Filter::Stream->new() ) if (defined($heap->{client})); $kernel->post( "ua" => "request", "got_response", $request ); } ### Handle HTTP responses from the POE::Component::Client::HTTP we've ### spawned at the beginning of the program. Send each response back ### to the client that requested it. Optionally display the response ### as text. sub handle_http_response{ my ( $kernel, $heap ) = @_[ KERNEL, HEAP ]; my $http_response = $_[ARG1]->[0]; my $chunk = $_[ARG1]->[1]; return if ((!$http_response) && (!$chunk)); our($sent_headers); our($CRLF,$is_text); unless ( ($sent_headers) ) { $sent_headers = 1; if ($http_response->content_type =~ /text|html/i){ $is_text = 1; print "Document is text\n"; } else{ # unless ($heap->{request}->header("X-IO-Error")){ $http_response->protocol('HTTP/1.0'); $http_response->remove_header("Content-Length") unless ($http_response->content_type); $heap->{client}->put($http_response->protocol . " " . $http_response->code . " (" . $http_response->message . ") " . $CRLF) if (defined($heap->{client})); print $http_response->code . " (" . $http_response->message . ") " . $http_response->protocol . $CRLF; $heap->{client}->put($http_response->headers_as_string($CRLF) . $CRLF) if (defined($heap->{client})); print $http_response->headers_as_string("\n") . "\n"; $is_text = 0; # } # else{ print "Continuing disrupted connection for " . $heap->{request}->uri . "\n"; } } } our $chunksent; our ($content, $totlen); $totlen = 0 if (not defined $totlen); if ((defined($chunk) && $chunk ne '-1')){ if (length($chunk) > 0 && $chunk ne '-1'){ $heap->{client}->put($chunk) if ((not $is_text) && defined($heap->{client})); print "Sent chunk of length " . length($chunk) . " bytes\n"; $totlen += length($chunk); if (!defined($heap->{client})){ $kernel->yield("shutdown"); } $content .= $chunk if ($is_text == 1 && $chunk ne '-1'); $totlen += length($chunk); } } else{ my $clen = length($content); #if ($http_response->header("Content-Length") > $totlen){ # Response was not fully received # $heap->{request}->header("Range","bytes=$clen-" . $http_response->header("Content-Length")); # $heap->{request}->header("X-IO-Error",1); # print "Resending request with range header (Content-Length = $clen) " . $heap->{request}->header("Range") . " for " . $heap->{request}->uri . "\n"; # $kernel->post( "streamua" => "request", "got_stream", $heap->{request} ); # Resubmit the request # return; #} $http_response->content($content); if ($http_response->header('Content-Encoding') =~ /gzip/i){ $http_response->content(Compress::Zlib::memGunzip($http_response->content)); $http_response->remove_header("Content-Encoding"); } if ($COMPRESS_TEXT && ($http_response->code == 200) && (lc($http_response->content_type) eq 'text/html') || (lc($http_response->content_type) eq 'text/plain') && ($heap->{request}->header("Accept-Encoding") =~ /gzip/i) && ($http_response->header("Content-Encoding") !~ /gzip/i)){ print "Length of content before gzip is " . length($http_response->content) . "\n"; $http_response->content(Compress::Zlib::memGzip($http_response->content)); $http_response->header("Content-Encoding","gzip"); } if ($is_text == 1){ $http_response->protocol('HTTP/1.0'); if (($http_response->code == 200) && ($http_response->content_type)){ use bytes (); $http_response->header('Content-Length',bytes::length($http_response->content)); } $heap->{client}->put($http_response->protocol . " " . $http_response->code " (" . $http_response->message . ") " . $CRLF) if (defined($heap->{client})); print $http_response->protocol . " " . $http_response->code . " (" . $http_response->message . ") " . $CRLF; $heap->{client}->put($http_response->headers_as_string($CRLF) . $CRLF) if (defined($heap->{client})); print $http_response->headers_as_string($CRLF) . $CRLF; $heap->{client}->put($http_response->content) if (defined($heap->{client})); print "Length of content at end of handle_http_response is " . length($http_response->content), "\n"; } $content = ''; $sent_headers = 0; $is_text = 0; $totlen = 0; $kernel->yield("shutdown"); } } ### Display requests and responses with brackets around them so they ### stand apart. sub display_thing { my $thing = shift; $thing =~ s/^/| /mg; print ",", '-' x 78, "\n"; print $thing; print "`", '-' x 78, "\n"; }