|
We have been using LVS with mon for many years to automatically put our
webservers in/out of rotation. It has been working great, but we are
upgrading the OS and various software including using the latest version
of heartbeat which comes with ldirectord. While functional, easy to
configure, and the fact it integrates with heartbeat easily, I found a
couple of important things lacking with ldirectord. Those things are
email alerts per virtual service and repeating email alerts while a
server remains inaccessible. I had seen requests and even an
implementation, but only using hard coded configuration settings in
ldirectord, on this mailing list, but nothing has come out of it after
suggestions were made to improve it. I would consider switching from
mon to ldirectord, but it would have to include the ability to send
email alerts.
I am not really a perl programmer, but I do know a little bit of perl,
so there are probably better ways of implementing these options, but I
have decided to write a small patch to include these options into
ldirectord version 1.131. I used Mail::Send, for sending the emails,
which also uses Mail::Mailer to find the first of several supported
email methods including sendmail and qmail. These are part of the
MailTools perl module. I have only tested this out on a newer slackware
distro, so there may or may not be problems on other distros. I have
been running this code for several days on a small non production test
setup and it seems to be working, although I have not tried to break it.
These configuration options are set per virtual service:
1) emailalert: A valid email address for sending alerts about the
changed connection status to any real server defined in the virtual service.
2) emailalertfreq: Delay in seconds between repeating email alerts
while any given real server in the virtual service remains
inaccessible. A setting of zero seconds will inhibit the repeating
alerts. Default setting is zero. The email timing accuracy of this
setting is dependent on the number of seconds defined in the
checkinterval configuration option.
The patch can also be found here:
http://www.on2.com/opensource/ldirectord-1.131-email-freq-alert.patch
Feel free to reply with any problems, suggestions, criticisms, or better
code.
anthony
--- /usr/sbin/ldirectord-1.131-20060131 2006-02-22 14:28:48.000000000 -0500
+++ /usr/sbin/ldirectord 2006-03-20 15:17:28.000000000 -0500
@@ -342,6 +342,21 @@
then it must be one of tcp or udp and will default to tcp. If a firewall
mark then the protocol must be fwm, which is the default.
+B<emailalert = ">I<emailaddress>B<">
+
+A valid email address for sending alerts about the changed connection
status
+to any real server defined in the virtual service. This option requires
+perl module MailTools to be installed. Automatically tries to send email
+using any of the built-in methods. See perldoc Mail::Mailer for more
info on
+methods.
+
+B<emailalertfreq => I<n>
+
+Delay in seconds between repeating email alerts while any given real server
+in the virtual service remains inaccessible. A setting of zero seconds
will
+inhibit the repeating alerts. Default setting is zero. The email timing
+accuracy of this setting is dependent on the number of seconds defined in
+the checkinterval configuration option.
=head1 FILES
@@ -400,6 +415,8 @@
@REAL
@VIRTUAL
$HOSTNAME
+ @EMAILSTATUS
+ @OLDEMAILSTATUS
$CRLF
);
@@ -784,6 +801,8 @@
$vsrv{negotiatetimeout} = 0;
$vsrv{num_connects} = 0;
$vsrv{httpmethod} = "GET";
+ $vsrv{emailalert} = "";
+ $vsrv{emailalertfreq} = 0;
push(@VIRTUAL, \%vsrv);
while(<CFGFILE>) {
$line++;
@@ -923,6 +942,12 @@
($1 eq "yes" || $1 eq "no")
or
&config_error($line, "quiescent must be 'yes' or 'no'");
$vsrv{quiescent} = $1;
+ } elsif ($rcmd =~ /^emailalert\s*=\s*(.*)/) {
+ $1 =~ /(.+)/ or &config_error($line, "no
email address specified");
+ $vsrv{emailalert} = $1;
+ } elsif ($rcmd =~
/^emailalertfreq\s*=\s*(\d*)/) {
+ $1 =~ /(\d+)/ or &config_error($line, "invalid email alert
frequency");
+ $vsrv{emailalertfreq} = $1;
} else {
&config_error($line, "Unknown command
$_");
}
@@ -1536,6 +1561,8 @@
my $nr;
my $server_down = {};
+ undef @EMAILSTATUS;
+
# read status of current ipvsadm -L -n
$oldsrv=&ld_read_ipvsadm();
@@ -1568,6 +1595,11 @@
for $nr (@$nreal) {
my $real_str = "$nr->{server}:$nr->{port}";
+ my %emailstat;
+ $emailstat{server}=$real_str . " " . $nv->{server} . ":" .
$nv->{port};
+ $emailstat{emailalertfreq}=$nv->{emailalertfreq};
+ $emailstat{emailalert}=$nv->{emailalert};
+ push(@EMAILSTATUS, \%emailstat);
if (! defined($or->{$real_str}) or
$or->{$real_str}->{weight} == 0) {
$server_down->{$real_str} = [$nv, $nr];
@@ -1727,6 +1759,14 @@
if (!check_cfgfile()) {
sleep $CHECKINTERVAL;
}
+
+ my $currenttime=time();
+ foreach my $es (@EMAILSTATUS){
+ if (($es->{alerttime} > 0 ) && ($currenttime - $es->{alerttime} >=
$es->{emailalertfreq})){
+ &ld_emailalert("Inaccessible real server: $es->{server}",
$es->{emailalert});
+ &ld_set_email_status($es->{server},
$currenttime);
+ }
+ }
}
}
@@ -2444,6 +2484,8 @@
$log_args .= "(" #. scalar(%{$v->{real_status}})
. " x $virtual_str)";
+ my $currentserver=$rservice . " " . $virtual_str;
+ my $currenttime=time();
if(defined($is_quiescent)) {
if (defined($or)) {
&system_wrapper("$IPVSADM -e "
@@ -2455,10 +2497,18 @@
. "$ipvsadm_args $rforw -w 0");
&ld_log("Quiescent $log_args (Weight set to 0)");
}
+ if(defined($$v{emailalert})) {
+ &ld_emailalert("Quiescent $log_args (Weight set to
0)",$$v{emailalert});
+ &ld_set_email_status($currentserver, $currenttime);
+ }
}
else {
&system_wrapper("$IPVSADM -d $ipvsadm_args");
&ld_log("Deleted $log_args");
+ if(defined($$v{emailalert})) {
+ &ld_emailalert("Deleted $log_args",$$v{emailalert});
+ &ld_set_email_status($currentserver, $currenttime);
+ }
}
}
@@ -2506,16 +2556,25 @@
if(defined($ov)){
$or=$ov->{"real"}->{$rservice};
}
+ my $currentserver=$rservice . " " . $v->{server} . ":" .
$v->{port};
if(defined($or)){
unless($or->{"weight"} eq $rwght and
get_forward_flag($or->{"forward"}) eq $rforw){
&system_wrapper("$IPVSADM -e $ipvsadm_args");
&ld_log("Restored $log_args (Weight set to $rwght)");
+ if(defined($$v{emailalert})) {
+ &ld_emailalert("Restored $log_args (Weight set to
$rwght)",$$v{emailalert});
+ &ld_set_email_status($currentserver, "0");
+ }
}
}
else {
&system_wrapper("$IPVSADM -a $ipvsadm_args");
&ld_log("Added $log_args (Weight set to $rwght)");
+ if(defined($$v{emailalert})) {
+ &ld_emailalert("Added $log_args (Weight set to
$rwght)",$$v{emailalert});
+ &ld_set_email_status($currentserver, "0");
+ }
}
}
@@ -2646,6 +2705,17 @@
my ($v, $r, $force) = @_;
if (! _status_down($v, $r) and ! defined($force)) {
+ my $currenttime=time();
+ foreach my $emailstats (@EMAILSTATUS) {
+ my $currentserver=$r->{server} . ":" . $r->{port} . " " .
$v->{server} . ":" . $v->{port};
+ if ($currentserver eq $emailstats->{server}){
+ if ($emailstats->{alertfreq} > 0){
+ if ($currenttime - $emailstats->{alerttime} >
$emailstats->{alertfreq}){
+ &ld_log("email alert freq: " .
$currentserver);
+ }
+ }
+ }
+ }
return;
}
@@ -2823,6 +2893,52 @@
return(0);
}
+# ld_emailalert
+# Send email alerts per virtual server
+# pre: message: Message to email
+# post: message is emailed if emailalert defined for virtualserver
+# return: 0 on success
+# 1 on error
+sub ld_emailalert
+{
+ my ($emailsubject,$emailto) = (@_);
+ require Mail::Send;
+ my $emailmsg;
+ my $emailfh;
+
+ unless ($emailmsg = new Mail::Send Subject=>$emailsubject, To=>$emailto
+ and $emailfh = $emailmsg->open
+ and print $emailfh ""
+ and $emailfh->close) {
+ &ld_log("failed to send email message\n");
+ return 1;
+ }
+ return(0);
+}
+
+# ld_set_email_status
+# Change the alert time for the real server
+# pre: real server and time setting
+# post: EMAILSTATUS array is updated
+# return: none
+sub ld_set_email_status{
+ my ($currentserver, $emailtime) = (@_);
+ @OLDEMAILSTATUS = @EMAILSTATUS;
+ undef @EMAILSTATUS;
+ foreach my $es (@OLDEMAILSTATUS){
+ my (%emailstats);
+ $emailstats{server}=$es->{server};
+ $emailstats{emailalertfreq}=$es->{emailalertfreq};
+ $emailstats{emailalert}=$es->{emailalert};
+ if ($currentserver eq $emailstats{server}){
+ $emailstats{alerttime}=$emailtime;
+ }else{
+ $emailstats{alerttime}=$es->{alerttime};
+ }
+ push(@EMAILSTATUS, \%emailstats);
+ }
+ return;
+}
# ld_debug
# Log a message to a STDOUT.
|
