一次DNS 故障引发的linux telnet 各端口缓慢的问题解决过程

昨天部署好了lvs+keepalived 并通过测试,  没有发现问题.今天上午忽然发现, 用ipvsadm –l  查看lvs信息,响应很慢,  然后去从LVS  telnet 节点的22号端口, 发现特别慢.   

开始我检查了一下keepalived.conf配置文件, 以为是同网段内多个lvs 设置,造成多播冲突,阻塞网络.  后来停止了lvs后故障依旧.  突然想到使用strace来分析, 一下找到了原因. 

telnet 命令 调用了如下共享对象库及文件, 说明telnet 先做权限和安全检查(如selinux) , 再做解析, 顺序是 , 先从hosts文件中找, 如果有对应的条目, 可直接接续, 如果没有,再从resolve.conf中找,   最后才反应成真正的IP地址.  

今天由于dns 202.96.209.5 故障, 所以连接其53端口有问题, 等待超时过了才能继续.  所以telnet有几秒钟的卡顿. 

解决方案:  删除resolve中的记录或指向可用的dns.

/etc/ld.so.preload

/etc/ld.so.cache

/usr/lib64/libkrb4.so.2

/usr/lib64/libkrb4.so.2

/usr/lib64/libdes425.so.3

/usr/lib64/libkrb5.so.3

/usr/lib64/libk5crypto.so.3

/lib64/libcom_err.so.2

/usr/lib64/libkrb5support.so.0

/lib64/libkeyutils.so.1

/lib64/libresolv.so.2

/usr/lib64/libncurses.so.5

/lib64/libselinux.so.1

/lib64/libdl.so.2

/lib64/libc.so.6

/lib64/libsepol.so.1

/etc/selinux/config

/etc/nsswitch.conf

/etc/resolv.conf

[root@lvs1 ~]# stracetelnet 10.67.130.232 22

execve("/usr/kerberos/bin/telnet",["telnet", "10.67.130.232", "22"], [/* 22 vars*/]) = 0

brk(0)                                  =0x2b867de32000

………………………………………………..

open("/etc/resolv.conf",O_RDONLY)      = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=24,...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000

read(3, "nameserver 202.96.209.5\n", 4096) = 24

read(3, "", 4096)                       = 0

close(3)                                = 0

munmap(0x2b8670400000, 4096)            = 0

uname({sys="Linux",node="lvs1", ...})  = 0

open("/etc/host.conf", O_RDONLY)        = 3

fstat(3, {st_mode=S_IFREG|0644, st_size=0,...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000

read(3, "", 4096)                       = 0

close(3)                                = 0

munmap(0x2b8670400000, 4096)            = 0

open("/etc/hosts", O_RDONLY)            = 3

fcntl(3, F_GETFD)                       = 0

fcntl(3, F_SETFD, FD_CLOEXEC)           = 0

fstat(3, {st_mode=S_IFREG|0644,st_size=168, ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000

read(3, "# Do not remove the followingli"..., 4096) = 168

read(3, "", 4096)                       = 0

close(3)                                = 0

munmap(0x2b8670400000, 4096)            = 0

open("/etc/ld.so.cache",O_RDONLY)      = 3

fstat(3, {st_mode=S_IFREG|0644,st_size=96055, ...}) = 0

mmap(NULL, 96055, PROT_READ, MAP_PRIVATE,3, 0) = 0x2b8670400000

close(3)                                = 0

open("/lib64/libnss_dns.so.2",O_RDONLY) = 3

read(3,"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\17\0\0\0\0\0\0"...,832) = 832

fstat(3, {st_mode=S_IFREG|0755,st_size=23736, ...}) = 0

mmap(NULL, 2113792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,3, 0) = 0x2b8672527000

mprotect(0x2b867252b000, 2093056,PROT_NONE) = 0

mmap(0x2b867272a000, 8192,PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) =0x2b867272a000

close(3)                                = 0

mprotect(0x2b867272a000, 4096, PROT_READ) =0

munmap(0x2b8670400000, 96055)           = 0

socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 3

connect(3, {sa_family=AF_INET, sin_port=htons(53),sin_addr=inet_addr("202.96.209.5")}, 28) = 0

fcntl(3, F_GETFL)                       = 0x2 (flags O_RDWR)

fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK)    = 0

poll([{fd=3, events=POLLOUT}], 1, 0)    = 1 ([{fd=3, revents=POLLOUT}])

sendto(3,"\255a\1\0\0\1\0\0\0\0\0\0\003232\003130\00267\00210\7in-ad"..., 44,MSG_NOSIGNAL, NULL, 0) = 44

poll([{fd=3, events=POLLIN}], 1, 5000)  = 0 (Timeout)

poll([{fd=3, events=POLLOUT}], 1, 0)    = 1 ([{fd=3, revents=POLLOUT}])

sendto(3,"\255a\1\0\0\1\0\0\0\0\0\0\003232\003130\00267\00210\7in-ad"..., 44,MSG_NOSIGNAL, NULL, 0) = 44

poll([{fd=3,events=POLLIN}], 1, 5000)  = 0 (Timeout)

close(3)                                = 0

fstat(1, {st_mode=S_IFCHR|0620,st_rdev=makedev(136, 2), ...}) = 0

mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000

write(1, "Trying 10.67.130.232...\r\n", 25Trying10.67.130.232...

) = 25

socket(PF_INET, SOCK_STREAM, IPPROTO_IP) =3

setsockopt(3, SOL_IP, IP_TOS, [16], 4)  = 0

connect(3, {sa_family=AF_INET, sin_port=htons(22),sin_addr=inet_addr("10.67.130.232")}, 16) = 0

open("/proc/filesystems",O_RDONLY)     = 4

read(4,"nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 4095) = 331

close(4)                                = 0

open("/root/.telnetrc",O_RDONLY)       = -1 ENOENT (No such fileor directory)

open("/proc/filesystems",O_RDONLY)     = 4

read(4,"nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 4095) = 331

close(4)                                = 0

write(1, "Connected to 10.67.130.232(10.6"..., 45Connected to 10.67.130.232 (10.67.130.232).

) = 45

本文出自 “可儿的爸爸” 博客，请务必保留此出处http://kerdady.blog.51cto.com/9695053/1595987