|
This happened again today, dead servers were not being removed. I had to
stop heartbeat, and allow the resources to transfer to the second load
balancer. Something is seriously wrong, but I don't know what it is. It
doesn't seem to happen on the second load balancer.
Also, one ftp service vip always goes to the fallback on the primary lvs,
but works correctly on the secondary (I hope this makes sense, it's early)
Is there possibly a bug in the ftp monitoring of ldirectord?
Michael
-----Original Message-----
From: Michael S. Moody [mailto:michael@xxxxxx]
Sent: Thursday, May 22, 2008 7:45 PM
To: lvs-users@xxxxxxxxxxxxxxxxxxxxxx
Subject: Re: [lvs-users] Dead servers not being removed from pool,
ldirectord
Here's the output of strace against the pid for about 30 seconds:
lvs1 fd # strace -p 10363
Process 10363 attached - interrupt to quit
restart_syscall(<... resuming interrupted call ...>) = 0
close(3) = 0
open("/tmp/PerlIO_rsrzrG", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY
(Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
unlink("/tmp/PerlIO_rsrzrG") = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fcntl(22, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in
progress)
select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0})
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = 0
fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(22, F_SETFL, O_RDWR) = 0
select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout)
close(22) = 0
stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) =
0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({3, 0}, {3, 0}) = 0
close(3) = 0
open("/tmp/PerlIO_9tbzOK", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY
(Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
unlink("/tmp/PerlIO_9tbzOK") = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fcntl(22, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in
progress)
select(24, NULL, [22], NULL, {0, 0}) = 0 (Timeout)
fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(22, F_SETFL, O_RDWR) = 0
close(22) = 0
stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) =
0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({3, 0}, {3, 0}) = 0
close(3) = 0
open("/tmp/PerlIO_IWMV4Y", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY
(Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
unlink("/tmp/PerlIO_IWMV4Y") = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fcntl(22, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in
progress)
select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0})
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = 0
fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(22, F_SETFL, O_RDWR) = 0
select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout)
close(22) = 0
stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) =
0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({3, 0}, {3, 0}) = 0
close(3) = 0
open("/tmp/PerlIO_ZQKIen", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY
(Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
unlink("/tmp/PerlIO_ZQKIen") = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fcntl(22, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in
progress)
select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0})
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = 0
fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(22, F_SETFL, O_RDWR) = 0
select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout)
close(22) = 0
stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) =
0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({3, 0}, {3, 0}) = 0
close(3) = 0
open("/tmp/PerlIO_FsEWhV", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea620) = -1 ENOTTY
(Inappropriate ioctl for device)
lseek(3, 0, SEEK_CUR) = 0
unlink("/tmp/PerlIO_FsEWhV") = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
open("/etc/protocols", O_RDONLY) = 22
fcntl(22, F_GETFD) = 0
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fstat(22, {st_mode=S_IFREG|0644, st_size=5681, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x2b31db397000
read(22, "# /etc/protocols\n#\n# Internet (I"..., 4096) = 4096
close(22) = 0
munmap(0x2b31db397000, 4096) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 22
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
ioctl(22, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffd25ea8c0) = -1 EINVAL
(Invalid argument)
lseek(22, 0, SEEK_CUR) = -1 ESPIPE (Illegal seek)
fcntl(22, F_SETFD, FD_CLOEXEC) = 0
fcntl(22, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(22, F_SETFL, O_RDWR|O_NONBLOCK) = 0
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = -1 EINPROGRESS (Operation now in
progress)
select(24, NULL, [22], NULL, {0, 0}) = 1 (out [22], left {0, 0})
connect(22, {sa_family=AF_INET, sin_port=htons(21),
sin_addr=inet_addr("192.168.1.195")}, 16) = 0
fcntl(22, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(22, F_SETFL, O_RDWR) = 0
select(24, [22], NULL, NULL, {0, 0}) = 0 (Timeout)
close(22) = 0
stat("/etc/ha.d/ldirectord.cf", {st_mode=S_IFREG|0644, st_size=8715, ...}) =
0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
rt_sigaction(SIGCHLD, NULL, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
nanosleep({3, 0}, ^C <unfinished ...>
Process 10363 detached
|
