1

这是我的代码的精简版本，显示了 webdriver 和 mitmproxy 的问题：

from libmproxy import controller
from libmproxy import proxy
import os
from multiprocessing import Process
from selenium import webdriver
from selenium.webdriver.common.proxy import *
import time


# traffic inception
class HttpMitmReporter(controller.Master):
    def __init__(self, proxy_server):
        controller.Master.__init__(self, proxy_server)

    def run(self):
        try:
            return controller.Master.run(self)
        except KeyboardInterrupt:
            self.shutdown()


# setup browser with proxy settings etc.
def setup_browser():
    global driver
    proper_setup = False

    proxy_con = "127.0.0.1:8088"
    my_proxy = Proxy({'proxyType': ProxyType.MANUAL,
                      'httpProxy': proxy_con,
                      'ftpProxy': proxy_con,
                      'sslProxy': proxy_con,
                      'noProxy': ''})

    while not proper_setup:
        try:
            driver = webdriver.Firefox(proxy=my_proxy)
            driver.set_page_load_timeout(20)
            driver.set_script_timeout(20)
            proper_setup = True
        except Exception as exception:
            print exception
            try:
                driver.quit()
            except Exception:
                time.sleep(3)


driver = None

# start proxy
config = proxy.ProxyConfig(
    cacert=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
server = proxy.ProxyServer(config, 8088)
reporter = HttpMitmReporter(server)
mitm_proxy = Process(target=reporter.run)
mitm_proxy.start()

setup_browser()

# urls to test
urls = [
        'http://support.microsoft.com/common/survey.aspx?showpage=1&scid=sw%3Bde%3B3663&altStyle=narrow&renderOption=OverrideDefault&fr=1',
        'http://outlook.com/',
        ]

# visit sites
for url in urls:
    print url
    try:
        driver.get(url)
        elements = driver.find_elements_by_tag_name('a')
    except Exception as ex:
        print ex
        driver.quit()
        setup_browser()

# terminate browser and proxy
driver.quit()
mitm_proxy.terminate()

如果执行此操作，您应该会看到 firefox 正在永远加载 Outlook.com，并且永远不会触发超时。
该问题仅发生在 mitmproxy 上。我测试了另一个代理，他们在这些 url 上工作，但我不能使用它们，因为它们要么不能检查 SSL 流量，要么只提供原始数据输出。但是我发现 mitmproxy 应该是最好的解决方案，如果它可以与 webdriver 一起正常工作。
此外，我已经使用在 shell 上启动的 mitmproxy 进行了测试 - > 相同的问题
以及使用 Java 的 webdriver - > 相同的问题
此外，这有时会发生在 fc2.com 等其他 url 上。

以下是我的配置的一些概述：
- Linux Mint Petra x64
- python 2.7
- Mitmproxy 0.10.1
- Selenium 2.41.0
- Firefox 28

有谁知道如何解决这个问题？
为什么 mitmproxy 会发生这种情况？
此外，我对使用 selenium 检查 http 和 https 流量的替代方法感兴趣。

使用 FlowMaster 更新：

from libmproxy import flow
from libmproxy import proxy
import os
from multiprocessing import Process
from selenium import webdriver
from selenium.webdriver.common.proxy import *
import time


# traffic inception
class HttpMitmReporter(flow.FlowMaster):

    def run(self):
        try:
            flow.FlowMaster.run(self)
        except KeyboardInterrupt:
            self.shutdown()

    def handle_request(self, r):
        f = flow.FlowMaster.handle_request(self, r)
        if f:
            r.reply()
        return f

    def handle_response(self, r):
        f = flow.FlowMaster.handle_response(self, r)
        if f:
            r.reply()
        return f


# setup browser with proxy settings etc.
def setup_browser():
    global driver
    proper_setup = False

    proxy_con = "127.0.0.1:8088"
    my_proxy = Proxy({'proxyType': ProxyType.MANUAL,
                      'httpProxy': proxy_con,
                      'ftpProxy': proxy_con,
                      'sslProxy': proxy_con,
                      'noProxy': ''})

    while not proper_setup:
        try:
            driver = webdriver.Firefox(proxy=my_proxy)
            driver.set_page_load_timeout(20)
            driver.set_script_timeout(20)
            proper_setup = True
        except Exception as exception:
            print exception
            try:
                driver.quit()
            except Exception:
                time.sleep(3)


driver = None

# start proxy
config = proxy.ProxyConfig(
    cacert=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
state = flow.State()
server = proxy.ProxyServer(config, 8088)
reporter = HttpMitmReporter(server, state)
mitm_proxy = Process(target=reporter.run)
mitm_proxy.start()

setup_browser()

# urls to test
urls = [
        'http://support.microsoft.com/common/survey.aspx?showpage=1&scid=sw%3Bde%3B3663&altStyle=narrow&renderOption=OverrideDefault&fr=1',
        'http://outlook.com/',
        ]

# visit sites
for url in urls:
    print url
    try:
        driver.get(url)
        elements = driver.find_elements_by_tag_name('a')
    except Exception as ex:
        print ex
        driver.quit()
        setup_browser()

# terminate browser and proxy
driver.quit()
mitm_proxy.terminate()
4

1 回答 1

0

看起来您错误地实例化了 mitmproxy。看看https://github.com/mitmproxy/mitmproxy/blob/master/examples/flowbasic，你肯定需要一个 FlowMaster 实例，而不是 controller.Master。

于 2014-05-03T08:42:42.493 回答