#!/usr/bin/perl # # servertest.pl # # A program to send periodic requests to one or more URLs # in order to get an indication of the average response time # and pattern of timeouts for each. # # Documentation is available in POD format in this file. # # (c) Copyright 2004 Software Garden, Inc. # All Rights Reserved. # Subject to Software License at the end of this file # use strict; # # NOTE: You must have this package: # use LWP::UserAgent; # Default values: my $progname = "ServerTest 1.0"; my $agentdefault = $progname; # Agent name in HTTP requests my $maxpolldefault = 1; # Only do one unless asked for more my $timeoutdefault = 30; # Timeout after about 30 seconds my $pollfrequencydefault = 3600; # poll once an hour by default # *** To avoid hitting a server too hard: *** my $pollfrequencymin = 10; # wait at least this long (seconds)... my $pollfrequencypollscutoff = 100; # ...if more than this many polls my $line; # # Define configuration information to get, consisting of which URLs to test, # where to save a log, etc.: # my %config_values; # most configuration values my @urls; # URLs to test my %specific_agent; # non-default agents hashed by URL they apply to $config_values{agent} = $agentdefault; $config_values{maxpolls} = $maxpolldefault; $config_values{pause} = "no"; $config_values{pollfrequency} = $pollfrequencydefault; $config_values{timeout} = $timeoutdefault; # # Get options. # # Start by going through each command line option and saving values. # If a configuration file is specified, then read it and # save values from there, then continue processing # command line arguments. # # The short definition of the options is in the "--help" text # here. There are more complete definitions in the pod definition # at the end of this file. # my %commandmap = (c => "config_file", l => "logfile", m => "maxpolls", p => "pollfrequency"); $ARGV[0] = "--interactive" unless @ARGV; # if no arguments, do interactive while (@ARGV) { if ($ARGV[0] eq '--help' || $ARGV[0] eq '-h') { print <<"EOF"; Usage: $0 [options] [URL1 URL2 ...] --agent "agent string" String to use in HTTP request with a name for the agent making the request (default: "$agentdefault") --config_file=filename, -c filename File from which to read option data --help, -h Show this help info --interactive Prompt for option values from the terminal (default if no options to the command) --logfile=filename, -l filename File in which to save optional log --maxpolls=n, -m n Number of tests for each URL (default: $maxpolldefault) --pause=yes/no If "yes" wait for user to press ENTER before exiting --pollfrequency=seconds, -p seconds Time between tests (default: $pollfrequencydefault) may not be less than $pollfrequencymin if maxpolls > $pollfrequencypollscutoff --restart=yes/no If not "no", parse log and start from where left off --test=URL URL to test (e.g.: test=www.domain.com) Multiple --test and --test_agent options allowed --test_agent=URL "agent string" URL to test with agent string for those requests only --timeout=seconds Max time to wait for each response (default: $timeoutdefault) Arguments that don't start with "-" are assumed to be URLs to test. Note: Commercial usage may require payment. See licensing terms. EOF exit; } elsif ($ARGV[0] =~ /^--agent$/) { shift @ARGV; $config_values{agent} = shift @ARGV; # NOTE!: uses --agent "string", not --agent=string } elsif ($ARGV[0] =~ /^--(config_file|logfile|maxpolls|pause|pollfrequency|restart|timeout)=(.*)/) { $config_values{$1} = $2; shift @ARGV; } elsif ($ARGV[0] =~ /^-(c|l|m|p)$/) { shift @ARGV; $config_values{$commandmap{$1}} = shift @ARGV; } elsif ($ARGV[0] =~ /^--test=(.*)/) { push @urls, $1; shift @ARGV; } elsif ($ARGV[0] =~ /^--test_agent=(.*)/) { push @urls, $1; shift @ARGV; $specific_agent{$1} = shift @ARGV; } elsif ($ARGV[0] =~ /^--(interactive)$/) { $config_values{$1} = 1; shift @ARGV; } elsif ($ARGV[0] =~ /^-/) { print <<"EOF"; Unknown option: $ARGV[0] For help, use: $0 --help EOF exit; } else { push @urls, shift @ARGV; } # # If --interactive was specified, do it now # if ($config_values{interactive}) { print <<"EOF"; $progname Note: Commercial usage may require payment. See licensing terms. Please provide the following option values. Default values are shown in [brackets] -- just press ENTER to accept the defaults. EOF print "Configuration File: (just press ENTER for none) "; $line = ; chomp $line; $config_values{config_file} = $line; print "Log to an output file [n]? (y/n) "; $line = ; chomp $line; if (lc($line) eq "y") { print "Logfile filename [$config_values{logfile}]: "; $line = ; chomp $line; $config_values{logfile} = $line || $config_values{logfile}; print "Restart polling from where left off in log [y]? (y/n) "; $line = ; chomp $line; $config_values{restart} = (lc($line) eq "n") ? "no" : "yes"; } else { $config_values{logfile} = ""; } print "Maxpolls (number of tests for each URL) [$config_values{maxpolls}]: "; $line = ; chomp $line; $config_values{maxpolls} = $line || $config_values{maxpolls}; print "Poll frequency (time between tests in seconds) [$config_values{pollfrequency}]: "; $line = ; chomp $line; $config_values{pollfrequency} = $line || $config_values{pollfrequency}; print "Timeout (max time to wait for each response in seconds) [$config_values{timeout}]: "; $line = ; chomp $line; $config_values{timeout} = $line || $config_values{timeout}; while (1) { print "URL to test, e.g., www.domain.com\n (just press ENTER when no more): "; $line = ; chomp $line; if ($line) { push @urls, $line; } else { last; } } print "Pause after running and await input before exiting [y]? (y/n) "; $line = ; chomp $line; $config_values{pause} = lc($line) eq "n" ? "no" : "yes"; print "\n"; $config_values{interactive} = ""; } # # If a config file was specified, process it now # # Format: # # name=value # optional comment # # Values are similar to the command line options. # For details, see the pod doc below. # if ($config_values{config_file}) { open (CONFIGFILE, $config_values{config_file}); while () { chomp; s/#.*//; s/^\s+//; s/\s+$//; next unless length; my ($var_name, $var_value) = split(/\s*=\s*/, $_, 2); if ($var_name eq "test") { my ($url_part, $agent_part) = split(" ", $var_value, 2); push @urls, $url_part; $specific_agent{$url_part} = $agent_part; } else { $config_values{$var_name} = $var_value; } } close CONFIGFILE; # Set config file name to null and get next option # More than one config file allowed $config_values{config_file} = ""; } } # Get next option # # Initialize and/or define a variety of values # my $pollnum; # for counting polls -- set above 1 if restarting by log scan code below my $pollfrequency = $config_values{pollfrequency} if defined $config_values{pollfrequency}; $pollfrequency = $pollfrequencymin if ($pollfrequency < $pollfrequencymin && $config_values{maxpolls} > $pollfrequencypollscutoff); # not too frequent -- that is very unfriendly my $request_time; # datetime test request made my $response_time; # datetime test responds my $res; # response to request my $rstatus; # text of response status my $toprint; # text to print and maybe log in a file my ($reqtmstr, $fetchtime, $timeoutpercent); my $restarted; # text flag for output header saying that log is a continuation # keep track of statistics for each URL being tested using these variables: my (%totalfetch, %maxfetch, $averagefetch, %numfetches, %numtimeouts); # # Read and process log if restarting. Refill accumulated statistics. # if (lc($config_values{restart}) ne "no" && $config_values{logfile}) { $restarted = " (restarted)"; open (PLOG, $config_values{logfile}); while () { chomp; # Split up line and do a simple parse: my ($indexv, $datev, $timev, $urlv, $fetchv, $restv) = split(/\s+/, $_, 6); if ($indexv =~ m/^\d+$/) { # valid entry $pollnum = $indexv if $indexv > $pollnum; # remember highest poll number if ($fetchv =~ m/^\d+$/) { # normal entry -- accumulate values $totalfetch{$urlv} += $fetchv; $numfetches{$urlv}++; $maxfetch{$urlv} = $fetchv if $fetchv > $maxfetch{$urlv}; } if ($fetchv =~ m/^FAILED/) { # Timeout of some sort $numtimeouts{$urlv}++; } } } close PLOG; } # # Test servers: Output header then enter main loop # print "$progname - n date time url fetchtime average max$restarted\n\n"; if ($config_values{logfile}) { # Append to log if requested open LOG, ">> $config_values{logfile}" or die "Couldn't open logfile $config_values{logfile}"; # Add startup line to log: print LOG "\n$progname - n date time url fetchtime average max$restarted\n\n"; close LOG; } # # Main loop: Make a request for each URL, wait $pollfrequency seconds, then do again # for (my $i=$pollnum+1; $i <= $config_values{maxpolls}; $i++) { foreach my $url (@urls) { # Setup request and remember time: my $req; $req = HTTP::Request->new(GET => ""); $req->header('Accept' => '*/*'); $req->uri("http://$url"); $request_time = time(); my ($sec, $min, $hour, $mday, $mon, $year) = localtime $request_time; $reqtmstr = sprintf("%04d-%02d-%02d %02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); # Create user agent to do request and make it: my $ua = LWP::UserAgent->new; $ua->timeout($config_values{timeout}); my $agent = $specific_agent{$url} || $config_values{agent}; $ua->agent($agent); $res = $ua->request($req); # When done, get time and status: $response_time = time(); $rstatus = $res->status_line; # if sucessful, accumulate stats on how long it took and create log entry: if ($res->is_success) { $fetchtime = $response_time - $request_time; $totalfetch{$url} += $fetchtime; # accumulate statistics $numfetches{$url}++; $averagefetch = $totalfetch{$url} / $numfetches{$url}; $maxfetch{$url} = $fetchtime if $fetchtime > $maxfetch{$url}; $timeoutpercent = 100 * $numtimeouts{$url} / ($numfetches{$url} + $numtimeouts{$url}); $toprint = sprintf("%d %s %24s %2d %4.1f (%d) %s", $i, $reqtmstr, $url, $fetchtime, $averagefetch, $maxfetch{$url}, ($numtimeouts{$url} ? sprintf("%d timeouts (%.1f%%)", $numtimeouts{$url}, $timeoutpercent) : "")); print "$toprint\n"; } # if not sucessful, accumulate different stats and log string: else { $rstatus =~ s/\n//g; $rstatus =~ s/\r//g; $toprint = sprintf("%d %s %24s FAILED: %s", $i, $reqtmstr, $url, $rstatus); print "$toprint\n"; $numtimeouts{$url}++; } # append to log if logfile given if ($config_values{logfile}) { # Append to log if requested open LOG, ">> $config_values{logfile}" or die "Couldn't open logfile $config_values{logfile}"; print LOG "$toprint\n"; close LOG; } } # on printed output, skip a line for readability # then wait appropriate amount of time to poll again print "\n"; sleep $pollfrequency unless ($i >= $config_values{maxpolls} || $pollfrequency < 1); } # # Wait for input if asked # if ($config_values{pause} eq "yes") { print "Press ENTER to exit: "; $line = ; } # # Done # __END__ =head1 NAME servertest.pl - log response times for a list of URLs over time =head1 SYNOPSIS servertest.pl --help --interactive --pause=yes/no --config_file=filename --logfile=filename --agent "agent string" --timeout=seconds --maxpolls=n --pollfrequency=seconds --restart=yes/no --test=URL --test_agent=URL "agent string" URL1 URL2 ... URLn =head1 DESCRIPTION This program sends periodic HTTP GET requests to one or more URLs in order to get an indication of the average response time and pattern of timeouts for each. The results of each poll are printed on STDOUT, and optionally logged to a text logfile. If a logfile is used, then each time the program is run it can "restart" by first processing the logfile so as to continue to accumulate statistics calculated with running totals. The output consists of lines in one of the following forms: n date time url fetchtime average (max) timeout-info or: n date time url FAILED: status line For example: 33 2004-01-27 19:30:23 www.dom314.com 1 0.8 (30) 33 2004-01-27 19:30:24 www.dom314.net FAILED: 500 read timeout 34 2004-01-27 19:45:26 www.dom314.com 0 0.8 (30) 34 2004-01-27 19:45:26 www.dom314.net 1 0.6 (10) 4 timeouts (1.3%) The fields are defined as follows: =over 5 =item B The sequence number of the poll. Each poll consists of making HTTP GET requests to each of the URLs listed for test in turn, in the order defined. If more than one URL is being tested, then there will be multiple lines with the same sequence number. If the program is run with "restart" not set to "no", then before the first poll it will read the logfile, if present, to find the highest sequence number and start counting after that. =item B The date and time of the request, local to the computer running the program. =item B The URL being tested. This value is right justified with spaces to help the numbers following it line up for easy reading. =item B The number of seconds taken to receive a successful response to the request. This is an integer. =item B The average time for a response to all of the polls for this URL in this sequence. Calculated by dividing the total time for all successful responses by the number of successful responses for this URL. Note that the number of polls to this URL may not correspond to the sequence number of the poll because this program may be restarted to add to an existing log but with a changed set of URLs to test. =item B The maximum time taken for a response in this sequence for this URL. =item B If there have been any failed requested for this URL, the total number of "timeouts" are listed, along with a percentage calculated by dividing the number of timeouts by the sum of the total successful requests and the total timeouts. =item B If the request fails for any reason, including a timeout or other error, then the word "FAILED:" is logged followed by the status information returned by the system, usually both as a number and some text. =back =head1 OPTIONS Options to this program may be set through command line options, interactive prompting and responses on the terminal, or the use of a configuration file. The command line options are processed in order. If an option that specifies interactive processing is encountered, then prompts are displayed and input will be awaited at that point in the processing of the command line options. If an option that specifies a configuration file is encountered, then that file is processed at that point, with processing then continued of the command line options. This lets you override configuration file values by specifying them after the configuration file on the command line or interactively. If no options are given on the command line, then interactive processing is assumed. =head2 Command Line Options: =over 5 =item B<--agent> "agent string" The string to use in the HTTP request with a name for the agent making the request. The default is "ServerTest 1.0". This value often ends up in server logs. By using a unique value like the default, it should be easy to filter out requests made by this program from normal, human browser requests so as not to mess up statistics. =item B<--config_file=>filename, B<-c> filename The file from which to read option data before processing any further command line options. The format of the file is defined below. Note that this command has both a short and long form. =item B<--help>, B<-h> Show help information listing the command options. =item B<--interactive> Start prompting for option values from the terminal (STDIN). This allows you to enter values without needing to edit the command line or have a configuration file. If there are no options or arguments on the command line, then "--interactive" is assumed. All values are prompted for, and then option processing on the command line continues. A configuration file specified interactively is read after all the prompting has finished. This option may appear more than once (for example, to specify a configuration file and then allow its values to be overridden). =item B<--logfile=>filename, B<-l> filename The name of the file in which to save optional log. This file will be created if it does not already exist. Each time the program starts, a heading line is added to the file. New log entries are appended to the end. If there is not an option with "restart=no", then the logfile is read (if it exists) and the accumulated values for each URL being tested are read in to let the calculations restart where they left off. The sequence number of the last poll executed is determined by looking for the highest poll sequence number in the logfile. If the filename is null (e.g., --logfile=, or -l ""), no logging will be done and the polling sequence counting will start from 1. =item B<--maxpolls=>n, B<-m> maxpolls The number of tests to be made for each URL. After all the requests are made for a sequence number equal to maxpolls, the program returns. The default is 1 (a single test). If you plan to run this program for an extended period of time, restarting after any system shutdowns, you may want to set this number very high. The program can usually be stopped early with ^C. =item B<--pause=>yes/no If "yes", the program will prompt for the user to press ENTER on the terminal (STDIN) before exiting. This is useful on systems, such as Windows, where the program window disappears after exit. The default is "no". =item B<--pollfrequency=>seconds, B<-p> seconds The time to wait between polls. After all of the URLs are tested in a single poll sequence, the program waits this number of seconds. The default is 3600 (poll once an hour). If the value is less than 10 seconds (it may be as low as zero), then the program will still wait 10 seconds if the "maxpolls" is greater than 100. This is to prevent the inadvertent unfriendly behavior of massive requests to a server. In the event that the program is being used on your own server and you want that type of behavior, you can set the $pollfrequencymin or $pollfrequencypollscutoff values in the program to override this check. =item B<--restart=>yes/no Set restart to "no" if you want the logfile to accumulate independent sequences of data. Leave it out, or set it to "yes" to initialize from the logfile before starting. =item B<--test=>URL A URL to test, without the access method (e.g., test=www.domain.com, or test=www.domain.com/file.html). Note that multiple --test and --test_agent options are allowed. In addition, any argument to the program that does not start with "-" is assumed to be a URL to test. =item B<--test_agent=>URL "agent string" A URL to test with agent string to be used for those requests only. =item B<--timeout=>seconds The maximum time to tell the system to wait for each response before logging it as an unsuccessful, failed request. The default is 30 seconds. =item B A URL to test, without the access method (e.g., "www.domain.com"). The same as the "--test" option. =back =head2 Configuration File Options Option values may be read from a configuration file. The filename is specified with the "--config_file" or "-c" command line options. The general format of the configuration file is as follows: # comment name1=value1 name2=value2 # optional comment Blank lines are ignored. Most options in the configuration file override the command line options encountered before the file is read, or are overridden by command line options encountered after, except "test" which adds to the list of URLs to test. The options that may be set are listed below. For a more detailed description of each option, see the Command Line Options descriptions above. =over 5 =item Bagent text The default value for Agent in request. =item Bfilename If present, where to save a log. =item Bn Quit after this many total poll sequences. =item Bseconds The amount of time to wait between each poll sequence. =item Byes/no Unless the option "restart=no" is present, parse logfile and start calculations from where we left off =item B=URL A URL to test with an HTTP GET request during each polling sequence, e.g., test=www.domain.com. Multiple test lines are allowed. =item BURL agent text The same as the "test" option, with the addition of a specific agent text for this URL to be used instead of the default. For example: test=www.domain.com Test by me@mydomain.com (The use of an email address lets the website owner find out who is doing this.) =item Bseconds The maximum time to wait for a response before considering it unsuccessful. =back =head1 EXAMPLES Here are three examples of invoking the program through the Perl system. It is assumed that the program is in the current directory and that the Perl system is in the search path. The first has all of the information on the command line and the second uses a configuration file. The third will result in prompts for the option values. Depending upon how the program is installed on your system, you may be able to execute it just by using "servertest" instead of "perl servertest.pl". In all cases, the options follow. perl servertest.pl -m 10 -p 5 "www.dom314.com" perl servertest.pl -c servertest.cfg -p 60 perl servertest.pl Here is an example of a configuration file: # # This is a sample configuration file # test=www.dom314.org test=www.dom314159.com logfile=servertest.log pollfrequency=900 # 15*60 = 15 minutes maxpolls=10000 # months... restart=yes =head1 VERSION This is servertest.pl v1.00. $Revision: 1.11 $ =head1 AUTHOR Dan Bricklin, Software Garden, Inc. =head1 COPYRIGHT (c) Copyright 2004 Software Garden, Inc. All Rights Reserved. See the Software License in the program file. =cut # # HISTORY # # Version 1.00 # $Date: 2004/05/18 01:57:48 $ # $Revision: 1.11 $ # # Dan Bricklin, Software Garden, Inc. (http://www.softwaregarden.com/) # -Initial version, including pod documentation. # # # TODO: # # ? # =begin license SOFTWARE GARDEN SERVERTEST SOFTWARE LICENSE This software is Copyright (c) Software Garden, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. This software may not be used for Commercial Purposes nor Distributed Commercially unless the terms of the companion Commercial License are met. The details of such license for this software, which includes payment requirements and the definitions of the terms "Commercial Purposes" and "Distributed Commercially", may be found on the Software Garden website, http://www.softwaregarden.com, or may be obtained from the postal address below. 2. Redistributions in source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 3. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 4. The end-user documentation included with the redistribution, if any, must include the following acknowledgment: "This product includes software developed by Software Garden, Inc. (www.softwaregarden.com)." Alternately, this acknowledgment may appear in the software itself, if and wherever such third-party acknowledgments normally appear. 5. The names Software Garden, Inc., and Dan Bricklin must not be used to endorse or promote products derived from this software without prior written permission from Software Garden, Inc., or Dan Bricklin, respectively. 6. The right to distribute this software or to use it for any purpose does not give you the right to use Servicemarks or Trademarks of Software Garden, Inc. or Dan Bricklin. 7. If any files are modified, you must cause the modified files to carry prominent notices stating that you changed the files and the date and nature of any change. 8. Modified versions that are redistributed must not remove rights or licensing notices present in the product, nor cause them not to be displayed when the software is executed. Disclaimer THIS SOFTWARE IS PROVIDED BY SOFTWARE GARDEN, INC., "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, WARRANTIES OF INFRINGEMENT AND THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SOFTWARE GARDEN, INC. OR DAN BRICKLIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE DISTRIBUTION OR USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Software Garden, Inc., and Dan Bricklin do not condone the use of this software for unlawful, malicious, harassing, disruptive, or abusive purposes. Additional Software In some cases this software is distributed with additional software copyright by others for the purpose of installation and execution. The above license for redistribution and modification does not apply to such additional software. License version: 1.00/2004-05-17 Software Garden, Inc. PO Box 610369 Newton Highlands, MA 02461 USA www.softwaregarden.com =end =cut