Re: Is it sslbot, or is it something else? I can only get up to endof May domains?

[Tom Brown]

> On Tue, 10 Jul 2001, Robert wrote:
> 
> > Perhaps there has been some changes that I've mised. Running sslbot doesn't
> > return the domains for June or July. I can only see up to May?
> >
> > Any ideas on what this could be from?
> >
> > thanks
> > Robert

Not sure what version you've got, nor when you went past 9 or 10 pages
which might actually be your issue... try the code below...

that said, is this program still usefull? I thought OpenSRS had
implemented a "download all domains" feature? I dump one of these
reports to disk everyday and "diff" the new report against the
previous one to see a summary of the day's activities... (I think
I copied that idea from Bill too.)

-Tom

#!/usr/bin/perl
#
# short script to grab all active domains from
# https://rr-n1-tor.opensrs.net/~vpop/resellers/
#
# Released to the public domain by Tom Brown, [EMAIL PROTECTED]
# (but I'd still like to have my name in the comments if you use it. :-)
# (additional work done by [EMAIL PROTECTED])
# (patched again in jan 2001, [EMAIL PROTECTED] to match
#     OpenSRS search-active-domains RWI changes)
#
# it will scan the pages of active domains...
# and dump the domains (one per line) 
# to the file: $RPT_PATH/$RPT_NAME.YYYYMMDD
#
# (set the variables RPT_PATH and RPT_NAME below :-)

my $USERID="xxxxxxxxx";  # your reseller id
my $PASS="XXXXXX";       # your reseller password

my $RPT_PATH=""; # report file path
my $RPT_NAME="opensrs.active"; # report filename (timestamp is appended)

#################################################

use strict;
use Net::SSLeay qw(get_https post_https sslcat make_headers make_form) ;

use vars qw($opt_c $opt_a $opt_d);
use Getopt::Std;

getopts('d');   
            # -d = debugging on

my $HOSTNAME="rr-n1-tor.opensrs.net";
my $URI="/~vpop/resellers/index";
my $ACTION="view_domains";  # used to be view_active ...

my $init_cookie = 'CheckCookie=CheckCookie'; # really should load the login page
my $AGENT='baremetal Reconciller';
my $headers=make_headers( 'User-Agent' => $AGENT );

# try to be robust, go load splash page to get check cookie if there is
# one...
my ($page, $response, %reply_headers)
                = get_https($HOSTNAME, 443, $URI, $headers,
                       make_form(''));
if ($opt_d) {
   print "headers from splash page:\n";
   foreach (keys %reply_headers) {
      printf "header: %s: %s\n", $_, $reply_headers{$_};
   }
   print "\n";
}
my $cookie = $reply_headers{'SET-COOKIE'} || ''; 
if ($cookie) {
   $headers=make_headers( 'User-Agent' => $AGENT, Cookie=>$cookie);
} else {
   warn "no \"check-cookie\"? Maybe OpenSRS has disabled this?\n";
}

## OK, try to login:

($page, $response, %reply_headers)
                = post_https($HOSTNAME, 443, $URI, $headers,
                       make_form(
                               'username'   => $USERID,
                               'password' => $PASS,
                               'action' => 'login'
                       ));

if ($page =~ m#Invalid username/password given#i) {
   print "OpenSRS says that's an Invalid userid/password.\n";
   exit;
}

if ($opt_d) {
   print "Auth results page headers:\n";
   foreach (keys %reply_headers) {
      printf "header: %s: %s\n", $_, $reply_headers{$_};
   }
   #print "\n$page\n" if $opt_d;
}

$cookie = $reply_headers{'SET-COOKIE'} or die "can't get auth cookie!";
print "cookie set to $cookie\n";

$headers=make_headers( 'User-Agent' => $AGENT,
                          Cookie => $cookie );

# open the output file with date stamp
my ($day, $month, $year) = (localtime)[3,4,5];
my $filedate = sprintf("%04d%02d%02d", $year+1900, $month+1, $day);
open F, ">$RPT_PATH" . $RPT_NAME . ".$filedate";
my $prior_line = '';

my @fetchpages = ('0');
my %validpages = (0=>1);
while (defined(my $fetchpage = shift(@fetchpages))) {
   ($page, $response, %reply_headers)
                = post_https($HOSTNAME, 443, $URI, $headers,
"action=view_domains&orderby=createdate&domain=&page=$fetchpage&name=&alpha=&from_day_c=All&from_day_x=All&from_month_c=All&from_month_x=All&from_year_x=All&from_year_c=All&to_day_c=All&to_day_x=All&to_month_c=All&to_month_x=All&to_year_x=All&to_year_c=All"
                       );
   die "whoa, request failed: $response" unless ($response);
   print "\npage $fetchpage is: $page\n\n" if ($opt_d);


#                    view_domains&orderby=createdate&domain=&page=$fetchpage&na
#   while ($page =~ m#view_domains&orderby=createdate&domain=&page=(\d+)&#g) {
   while ($page =~ m#view_domains&orderby=createdate[0-9a-z_A-Z&=]+page=(\d+)(&|")#g) {
      my $p = $1;
      if (!defined($validpages{$p})) {
         push(@fetchpages,$p); # schedule it
         $validpages{$p} = 1;  # record the fact it is scheduled
      }
   }

   print "fetched page $fetchpage\n";

   $page =~ s/^.* END HEADER -->//s; # throw away page header...

   print "stripped header\n" if ($opt_d);

   my $found = 0;
   # note the embedded action below is not $ACTION it is the singular form
   #                               reg                 exp
   while ($page =~ 
m#<tr>\s*<td[^>]*>(.*?)</td>\s*<td[^>]*>(.*?)</td>\s*.*?action=view_domain&name=(.*?)">.*?</tr>#igs)
 {


      my $this_line = "$3: $1: $2";

print qq~$this_line = "$3: $1: $2" \n~ if ($opt_d);

      print F "$this_line\n" if ($prior_line ne $this_line);
      $prior_line = $this_line;
      $found = 1;
   }
   die "Yikes, I couldn't parse any domains out of this page:\n$page\n"
     unless $found;
}

close F;