修改Python的pyxmpp2中的主循环使其提高性能
引子
之前clubot使用的pyxmpp2的默认mainloop也就是一个poll的主循环,但是clubot上线后资源占用非常厉害,使用strace跟踪发现clubot在不停的poll,查看pyxmpp2代码发现pyxmpp2的poll在使用超时阻塞时使用最小超时时间,而最小超时时间一直是0,所以会变成一个没有超时的非阻塞poll很浪费资源,不打算更改库代码,所以自己仿照poll的mainloop写了一个更加高效的epoll的mainloop
实现
#!/usr/bin/envpython
#-*-coding:utf-8-*-
#
#Author:cold
#E-mail:wh_linux@126.com
#Date:13/01/0610:41:31
#Desc:Clubotepollmainloop
#
from__future__importabsolute_import,division
importselect
frompyxmpp2.mainloop.interfacesimportHandlerReady,PrepareAgain
frompyxmpp2.mainloop.baseimportMainLoopBase
fromplugin.utilimportget_logger
classEpollMainLoop(MainLoopBase):
"""Maineventloopbasedontheepoll()syscallonLinuxsystem"""
READ_ONLY=(select.EPOLLIN|select.EPOLLPRI|select.EPOLLHUP|
select.EPOLLERR|select.EPOLLET)
READ_WRITE=READ_ONLY|select.EPOLLOUT
def__init__(self,settings=None,handlers=None):
self.epoll=select.epoll()
self._handlers={}
self._unprepared_handlers={}
self._timeout=None
self._exists_fd={}
self.logger=get_logger()
MainLoopBase.__init__(self,settings,handlers)
return
def_add_io_handler(self,handler):
self._unprepared_handlers[handler]=None
self._configure_io_handler(handler)
def_configure_io_handler(self,handler):
ifself.check_events():
return
ifhandlerinself._unprepared_handlers:
old_fileno=self._unprepared_handlers[handler]
prepared=self._prepare_io_handler(handler)
else:
old_fileno=None
prepared=True
fileno=handler.fileno()
ifold_filenoisnotNoneandfileno!=old_fileno:
delself._handlers[old_fileno]
self._exists.pop(old_fileno,None)
self.epoll.unregister(old_fileno)
ifnotprepared:
self._unprepared_handlers[handler]=fileno
ifnotfileno:
return
self._handlers[fileno]=handler
events=0
ifhandler.is_readable():
events|=self.READ_ONLY
ifhandler.is_writable():
events|=self.READ_WRITE
ifevents:
iffilenoinself._exists_fd:
self.epoll.modify(fileno,events)
else:
self._exists_fd.update({fileno:1})
self.epoll.register(fileno,events)
def_prepare_io_handler(self,handler):
ret=handler.prepare()
ifisinstance(ret,HandlerReady):
delself._unprepared_handlers[handler]
prepared=True
elifisinstance(ret,PrepareAgain):
ifret.timeoutisnotNone:
ifself._timeoutisnotNone:
self._timeout=min(self._timeout,ret.timeout)
else:
self._timeout=ret.timeout
prepared=False
else:
raiseTypeError("Unexpectedresultfromprepare()")
returnprepared
def_remove_io_handler(self,handler):
ifhandlerinself._unprepared_handlers:
old_fileno=self._unprepared_handlers[handler]
delself._unprepared_handlers[handler]
else:
old_fileno=handler.fileno()
ifold_filenoisnotNone:
try:
delself._handlers[old_fileno]
self._exists.pop(old_fileno,None)
self.epoll.unregister(old_fileno)
exceptKeyError:
pass
defloop_iteration(self,timeout=60):
next_timeout,sources_handled=self._call_timeout_handlers()
ifself.check_events():
return
ifself._quit:
returnsources_handled
forhandlerinlist(self._unprepared_handlers):
self._configure_io_handler(handler)
ifself._timeoutisnotNone:
timeout=min(timeout,self._timeout)
ifnext_timeoutisnotNone:
timeout=min(next_timeout,timeout)
iftimeout==0:
timeout+=1#带有超时的非阻塞,解约资源
events=self.epoll.poll(timeout)
forfd,flaginevents:
ifflag&(select.EPOLLIN|select.EPOLLPRI|select.EPOLLET):
self._handlers[fd].handle_read()
ifflag&(select.EPOLLOUT|select.EPOLLET):
self._handlers[fd].handle_write()
ifflag&(select.EPOLLERR|select.EPOLLET):
self._handlers[fd].handle_err()
ifflag&(select.EPOLLHUP|select.EPOLLET):
self._handlers[fd].handle_hup()
#ifflag&select.EPOLLNVAL:
#self._handlers[fd].handle_nval()
sources_handled+=1
self._configure_io_handler(self._handlers[fd])
returnsources_handled
使用
如何使用新的mainloop?只需在实例化Client时传入
mainloop=EpollMainLoop(settings) client=Client(my_jid,[self,version_provider],settings,mainloop)
这样就会使用epoll作为mainloop
注意
epoll仅仅在Linux下支持