1

我正在尝试编写一个小程序来发送和接收 UDP 流量并通过 HTTP 接口接收命令。HTTP 服务器合二为一multiprocessing.Process；UDP服务器位于另一个。这两个进程通过 python 进行通信multiprocessing.Pipe。我在下面附上了完整的代码。

我有2个相关问题：

  • 如何在 python 中使用 kqueue 处理多个文件描述符/kevents（套接字文件描述符有效，管道文件描述符似乎不 - 不确定我使用的管道是否等同于文件）？
  • 如何区分这些 kevents，以便在读取管道和套接字时应用不同的功能？

我希望我的 UDP 服务器执行的伪代码：

kq = new kqueue
udpEvent = kevent when socket read
pipeEvent = kevent when pipe read
while:
    for event in kq.conrol([udpEvent, pipeEvent]):
        if event == udpEvent:
             # do something
        elif event == pipeEvent:
             print "HTTP command via pipe:", pipe.recv()

现在，UDP 服务器识别套接字事件并正确读取套接字。但是，当我将管道 kevent 添加到 kqueue 时，程序会不停地吐出管道事件。我将过滤器设置为已写入管道，但我假设 1）这是错误的 2）更具体地说，pythonmultiprocessing.Pipe就像一个常规的 unix 管道，需要以不同的方式处理。

.....
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>
<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 ^C<select.kevent ident=4297866384 filter=-29216 flags=0x4000 fflags=0x1 data=0x16 udata=0x4000000000000>

主文件

import sys
from multiprocessing import Process, Pipe
# from userinterface import OSXstatusbaritem # use like so: OSXstatusbaritem.start(pipe)
from server import Server
import handler # UI thingy

# For UI, use simple HTTP server with various endpoints
# open a connection: localhost:[PORT]/open/[TARGET_IP]

def startServer(pipe):
    UDP_IP = "127.0.0.1"
    UDP_PORT = 9000

    print "starting server"
    s = Server(pipe)
    s.listen(UDP_IP, UDP_PORT)
    print "finishing server"

import BaseHTTPServer
def startUI(pipe):
    HTTP_PORT = 4567
    server_class = BaseHTTPServer.HTTPServer
    myHandler = handler.handleRequestsUsing(pipe)
    httpd = server_class(('localhost', 4567), myHandler)
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
          pass
    httpd.server_close()

def main():
    # Named full duplex pipe for communicating between server process and UI
    pipeUI, pipeServer = Pipe()

    # Start subprocesses
    pServer = Process(target=startServer, args=(pipeServer,))
    pServer.start()
    startUI(pipeUI)
    pServer.join()

if __name__ == "__main__": sys.exit(main())

server.py (UDP)

import sys
import select # for kqueue
from socket import socket, AF_INET, SOCK_DGRAM
from multiprocessing import Process, Pipe

class Server:
    def __init__(self, pipe):
        self.pipe = pipe

    def listen (self, ipaddress, port):
        print "starting!"

        # Initialize listening UDP socket
        sock = socket(AF_INET, SOCK_DGRAM)
        sock.bind((ipaddress, port))

        # Configure kqueue
        kq = select.kqueue()
        # Event for UDP socket data available
        kevent0 = select.kevent( sock.fileno(),
                                 filter=select.KQ_FILTER_READ,
                                 flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR)
        # Event for message queue from other processes (ui)
        kevent1 = select.kevent( self.pipe.fileno(),
                                 filter=select.KQ_FILTER_WRITE,
                                 flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE)                        

        # TODO: Figure out how to handle multiple kevents on kqueue
        # TODO: Need an event for TUN data

        # Start kqueue      
        while True:
            revents = kq.control([kevent0, kevent1], 1, None)
            for event in revents:
                print event
        kq.close()
        # close file descriptors (os.close(fd))

handler.py（HTTP 接口）

import BaseHTTPServer

# Simple HTTP endpoints for controlling prototype Phantom implementation.
# The following commands are supported:
# 1. Open a connection via /open/[IP]:[PORT]
# 2. ????

class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    pipe = None

    def __init__(self, pipe, *args):
        RequestHandler.pipe = pipe
        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args)

    def do_HEAD(s):
        s.send_response(200)
        s.send_header("Content-type", "application/json")
        s.end_headers()
    def do_GET(s):
        s.send_response(200)
        s.send_header("Content-type", "application/json")
        s.end_headers()

        # Open connection command
        if s.path.startswith('/open/'):
            addrStr = s.path[6:len(s.path)]
            (address, port) = tuple(filter(None, addrStr.split(':')))
            port = int(port)
            print "opening address: ", address, "port:", port
            RequestHandler.pipe.send(['open', address, port])

def handleRequestsUsing(logic):
    return lambda *args: RequestHandler(logic, *args)

更新：

我用 select 重写了服务器监听方法。对于一个不会使用超过 3 或 4 个 fds 的慢速 Python 原型，速度无论如何都无关紧要。Kqueue 将成为另一天的主题。

def listen (self, ipaddress, port): print "starting!"

# Initialize listening non-blocking UDP socket
sock = socket(AF_INET, SOCK_DGRAM)
sock.setblocking(0)
sock.bind((ipaddress, port))

inputs = [sock, self.pipe] # stuff we read
outputs = [] # stuff we expect to write
while inputs:
    readable, writable, exceptional = select.select(inputs, outputs, inputs)

    for event in readable:
        if event is sock:
            self.handleUDPData( sock.recvfrom(1024) )
        if event is self.pipe:
            print "pipe event", self.pipe.recv()
4

1 回答 1

3

我知道这是一个老问题，但我可以给你一个用于多线程 HTTP 服务器的 kqueue 套接字轮询示例，这是我在阅读 C 源代码和 kqueue 手册页后发现的。

#bsd socket polling
#I make all the relevant flags more C like to match the kqueue man pages
from select import kevent, kqueue
from select import KQ_EV_ADD as EV_ADD, KQ_EV_ONESHOT as EV_ONESHOT   
from select import KQ_EV_EOF as EV_EOF

from .common import Client_Thread #a parent class who's implementation is irrelevant to the question, lol

class BSD_Client(Client_Thread):
    def __init__(self, *args):
        Client_Thread.__init__(self, *args)
        #Make a kqueue object for the thread 
        kq = kqueue()
        #Make a one-shot kev for this kqueue for when the kill socket is
        #connected to. The connection is only made once, so why not tell
        #that to our kqueue? The default filter is EVFILT_READ, so we don't
        #need to specify that. The default flag is just EV_ADD.
        kill_kev = kevent(self.kill_fd, flags=EV_ADD|EV_ONESHOT)
        #using defaults for the client socket.
        client_kev = kevent(self.client_sock)
        #we only need to keep track of the kqueue's control func.
        #This also makes things prettier in the run func.
        self.control = kq.control
        #now, we add thel list of events we just made to our kqueue.
        #The first 0 means we want a list of at most 0 length in return.
        #the second 0 means we want no timeout (i.e. do this in a 
        #non-blocking way.) 
        self.control([client_kev, kill_kev], 0, 0)

    def run(self):
        while True:
            #Here we poll the kqueue object.
            #The empty list means we are adding no new events to the kqueue.
            #The one means we want a list of at most 1 element. Then None
            #Means we want block until an event is triggered.
            events = self.control([], 1, None)
            #If we have an event, and the event is for the kill socket 
            #(meaning somebody made a connection to it), then we break the 
            #loop and die.
            if events and events[0].ident == self.kill_fd:
                self.die()
                break
            #If all that is left is an EOF in our socket, then we break
            #the loop and die. Kqueues will keep returning a kevent
            #that has been read once, even when they are empty.
            if events and events[0].flags & EV_EOF:
                self.die()
                break
            #Finally, if we have an event that isn't for the kill socket and
            #does not have the EOF flag set, then there is work to do. If
            #the handle client function (defined in the parent class) returns
            #1, then we are done serving a page and we can die.
            if events and self.handle_client():
                self.die()
                break
        client.close()

self.die 所做的只是put将客户端 ip:port 字符串放到用于消息传递的队列中。一个不同的线程gets 来自队列的字符串，打印一条消息并joins 相关的线程对象。当然，我没有为此使用管道，仅使用套接字。我确实在 kqueue 的在线联机帮助页上找到了这个

Fifos, Pipes
有数据读取时返回；data 包含
可用的字节数。

当最后一个写入器断开连接时，过滤器将在
标志中设置 EV_EOF。这可以通过传入 EV_CLEAR 来清除，此时过滤器 将
继续等待数据可用，然后再返回

因此，也许在您的 udp 服务器中，您循环浏览 revents 列表，您应该按照手册页中的说明进行操作吗？实际上，您甚至不需要遍历最长为 1 的列表。也许你的监听功能应该是这样的......

def listen(self, ip, port):
    print "Starting!"
    sock = socket.socket(AF_INET, SOCK_DGRAM)
    sock.bind((ip, port))
    kq = select.kqueue()
    kev0 = select.kevent(sock)
    kev1 = select.kevent(self.pipe)
    kq.control([kev0, kev1], 0, 0)
    while True: #this loop never breaks! so this whole function blocks forever like this
        revents = kq.control([], 1, None)
        if revents:
            event = revents[0]
            if event.flags & select.KQ_EV_EOF:
                new_event = select.kevent(event.ident, flags=select.KQ_EV_CLEAR)
                kq.control([new_event], 0, 0)
            else:
                print event

我真的建议按照我的方式导入标志和函数，它使它更类似于您必须比较的基于 C 的联机帮助页，而且我认为它看起来更漂亮。我还想指出，我的课程与您的课程有些不同，因为每个新客户端都会获得这个实例，并且每个客户端都将在自己的线程中运行。

于 2014-03-20T18:08:59.483 回答