diff --git a/docs/_extensions/apilinks.py b/docs/_extensions/apilinks.py index 656d350..cf3c3c7 100644 --- a/docs/_extensions/apilinks.py +++ b/docs/_extensions/apilinks.py @@ -23,7 +23,7 @@ def make_api_link(name, rawtext, text, lineno, inliner, full_name = full_name.strip() label = label.strip('>').strip() else: - full_name = label = text + full_name = text #get the base url for api links from the config file env = inliner.document.settings.env diff --git a/docs/projects/core/examples/index.rst b/docs/projects/core/examples/index.rst index d3ff699..67c8371 100644 --- a/docs/projects/core/examples/index.rst +++ b/docs/projects/core/examples/index.rst @@ -175,6 +175,9 @@ POSIX Specific Tricks Miscellaneous ------------- + + + - :download:`shaper.py` - example of rate-limiting your web server - :download:`stdiodemo.py` - example using stdio, Deferreds, LineReceiver and twisted.web.client. @@ -196,4 +199,7 @@ Miscellaneous protocols to display fix data as it is received from the device - :download:`wxacceptance.py` - acceptance tests for wxreactor - :download:`postfix.py` - test application for PostfixTCPMapServer -- :download:`udpbroadcast.py` - broadcasting using UDP + + + + diff --git a/docs/projects/core/examples/udpbroadcast.py b/docs/projects/core/examples/udpbroadcast.py deleted file mode 100644 index 0a02a56..0000000 --- a/docs/projects/core/examples/udpbroadcast.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -An example demonstrating how to send and receive UDP broadcast messages. - -Every second, this application will send out a PING message with a unique ID. -It will respond to all PING messages with a PONG (including ones sent by -itself). You can tell how many copies of this script are running on the local -network by the number of "RECV PONG". - -Run using twistd: - -$ twistd -ny udpbroadcast.py -""" - -from uuid import uuid4 - -from twisted.application import internet, service -from twisted.internet.protocol import DatagramProtocol -from twisted.python import log - - - -class PingPongProtocol(DatagramProtocol): - noisy = False - - def __init__(self, controller, port): - self.port = port - - - def startProtocol(self): - self.transport.setBroadcastAllowed(True) - - - def sendPing(self): - pingMsg = "PING {0}".format(uuid4().hex) - self.transport.write(pingMsg, ('', self.port)) - log.msg("SEND " + pingMsg) - - - def datagramReceived(self, datagram, addr): - if datagram[:4] == "PING": - uuid = datagram[5:] - pongMsg = "PONG {0}".format(uuid) - self.transport.write(pongMsg, ('', self.port)) - log.msg("RECV " + datagram) - elif datagram[:4] == "PONG": - log.msg("RECV " + datagram) - - - -class Broadcaster(object): - - def ping(self, proto): - proto.sendPing() - - - def makeService(self): - application = service.Application('Broadcaster') - - root = service.MultiService() - root.setServiceParent(application) - - proto = PingPongProtocol(controller=self, port=8555) - root.addService(internet.UDPServer(8555, proto)) - root.addService(internet.TimerService(1, self.ping, proto)) - - return application - - -application = Broadcaster().makeService() diff --git a/docs/projects/core/howto/index.rst b/docs/projects/core/howto/index.rst index 7c892d7..d2dd324 100644 --- a/docs/projects/core/howto/index.rst +++ b/docs/projects/core/howto/index.rst @@ -153,7 +153,8 @@ Developer Guides Add some security to your network transport. - :doc:`UDP Networking ` - How to use Twisted's UDP implementation, including multicast and broadcast functionality. + + Multicast too! - :doc:`Using processes ` Launching sub-processes, the correct way. diff --git a/docs/projects/core/howto/threading.rst b/docs/projects/core/howto/threading.rst index 95d2298..fcb1fdc 100644 --- a/docs/projects/core/howto/threading.rst +++ b/docs/projects/core/howto/threading.rst @@ -6,20 +6,33 @@ Using Threads in Twisted ======================== + + + + + Running code in a thread-safe manner ------------------------------------ -Most code in Twisted is not thread-safe. -For example, writing data to a transport from a protocol is not thread-safe. -Therefore, we want a way to schedule methods to be run in the main event loop. -This can be done using the function :api:`twisted.internet.interfaces.IReactorThreads.callFromThread `:: - from twisted.internet import reactor + +Most code in Twisted is not thread-safe. For example, +writing data to a transport from a protocol is not thread-safe. +Therefore, we want a way to schedule methods to be run in the +main event loop. This can be done using the function :api:`twisted.internet.interfaces.IReactorThreads.callFromThread ` : + + + +.. code-block:: python + + + from twisted.internet import reactor + def notThreadSafe(x): """do something that isn't thread-safe""" # ... - + def threadSafeScheduler(): """Run in thread-safe manner.""" reactor.callFromThread(notThreadSafe, 3) # will run 'notThreadSafe(3)' @@ -27,74 +40,127 @@ This can be done using the function :api:`twisted.internet.interfaces.IReactorTh reactor.run() + + + Running code in threads ----------------------- -Sometimes we may want to run methods in threads. -For example, in order to access blocking APIs. -Twisted provides methods for doing so using the :api:`twisted.internet.interfaces.IReactorThreads ` API. -Additional utility functions are provided in :api:`twisted.internet.threads `. -Basically, these methods allow us to queue methods to be run by a thread pool. -For example, to run a method in a thread we can do:: + +Sometimes we may want to run methods in threads - for +example, in order to access blocking APIs. Twisted provides +methods for doing so using the IReactorThreads API (:api:`twisted.internet.interfaces.IReactorThreads ` ). +Additional utility functions are provided in :api:`twisted.internet.threads ` . Basically, these +methods allow us to queue methods to be run by a thread +pool. + + + + +For example, to run a method in a thread we can do: - from twisted.internet import reactor + + +.. code-block:: python + + + from twisted.internet import reactor + def aSillyBlockingMethod(x): import time time.sleep(2) print x - + # run method in thread reactor.callInThread(aSillyBlockingMethod, "2 seconds have passed") reactor.run() + + + Utility Methods --------------- -The utility methods are not part of the :api:`twisted.internet.reactor ` APIs, but are implemented in :api:`twisted.internet.threads `. -If we have multiple methods to run sequentially within a thread, we can do:: + +The utility methods are not part of the :api:`twisted.internet.reactor ` APIs, but are implemented +in :api:`twisted.internet.threads ` . + + - from twisted.internet import reactor, threads +If we have multiple methods to run sequentially within a thread, +we can do: + + + + + +.. code-block:: python + + + from twisted.internet import reactor, threads + def aSillyBlockingMethodOne(x): import time time.sleep(2) print x - + def aSillyBlockingMethodTwo(x): print x - + # run both methods sequentially in a thread commands = [(aSillyBlockingMethodOne, ["Calling First"], {})] commands.append((aSillyBlockingMethodTwo, ["And the second"], {})) threads.callMultipleInThread(commands) reactor.run() -For functions whose results we wish to get, we can have the result returned as a Deferred:: - from twisted.internet import reactor, threads + +For functions whose results we wish to get, we can have the +result returned as a Deferred: + + + + +.. code-block:: python + + + from twisted.internet import reactor, threads + def doLongCalculation(): # .... do long calculation here ... return 3 - + def printResult(x): print x - + # run method in thread and get result as defer.Deferred d = threads.deferToThread(doLongCalculation) d.addCallback(printResult) reactor.run() -If you wish to call a method in the reactor thread and get its result, you can use :api:`twisted.internet.threads.blockingCallFromThread `:: + + +If you wish to call a method in the reactor thread and get its result, +you can use :api:`twisted.internet.threads.blockingCallFromThread ` : + + + + + +.. code-block:: python + + from twisted.internet import threads, reactor, defer from twisted.web.client import getPage from twisted.web.error import Error - + def inThread(): try: result = threads.blockingCallFromThread( @@ -104,25 +170,54 @@ If you wish to call a method in the reactor thread and get its result, you can u else: print result reactor.callFromThread(reactor.stop) - + reactor.callInThread(inThread) reactor.run() -``blockingCallFromThread`` will return the object or raise the exception returned or raised by the function passed to it. -If the function passed to it returns a Deferred, it will return the value the Deferred is called back with or raise the exception it is errbacked with. + + + +``blockingCallFromThread`` will return the object or raise +the exception returned or raised by the function passed to it. If the +function passed to it returns a Deferred, it will return the value the +Deferred is called back with or raise the exception it is errbacked +with. + + + Managing the Thread Pool ------------------------ -The thread pool is implemented by :api:`twisted.python.threadpool.ThreadPool `. -We may want to modify the size of the thread pool, increasing or decreasing the number of threads in use. -We can do this do this quite easily:: + +The thread pool is implemented by :api:`twisted.python.threadpool.ThreadPool ` . + + + + +We may want to modify the size of the threadpool, increasing +or decreasing the number of threads in use. We can do this +do this quite easily: - from twisted.internet import reactor + + + +.. code-block:: python + + + from twisted.internet import reactor + reactor.suggestThreadPoolSize(30) -The default size of the thread pool depends on the reactor being used; the default reactor uses a minimum size of 5 and a maximum size of 10. -Be careful that you understand threads and their resource usage before drastically altering the thread pool sizes. + + + +The default size of the thread pool depends on the reactor being used; +the default reactor uses a minimum size of 5 and a maximum size of 10. Be +careful that you understand threads and their resource usage before +drastically altering the thread pool sizes. + + diff --git a/docs/projects/core/howto/udp.rst b/docs/projects/core/howto/udp.rst index d43d7b5..a68bf6f 100644 --- a/docs/projects/core/howto/udp.rst +++ b/docs/projects/core/howto/udp.rst @@ -6,165 +6,301 @@ UDP Networking ============== + + + + + Overview -------- -Unlike TCP, UDP has no notion of connections. -A UDP socket can receive datagrams from any server on the network and send datagrams to any host on the network. -In addition, datagrams may arrive in any order, never arrive at all, or be duplicated in transit. -Since there are no connections, we only use a single object, a protocol, for each UDP socket. -We then use the reactor to connect this protocol to a UDP transport, using the :api:`twisted.internet.interfaces.IReactorUDP ` reactor API. + +Unlike TCP, UDP has no notion of connections. A UDP socket can receive +datagrams from any server on the network and send datagrams to any host on +the network. In addition, datagrams may arrive in any order, never arrive at +all, or be duplicated in transit. + + + + +Since there are no connections, we only use a single object, a protocol, +for each UDP socket. We then use the reactor to connect this protocol to a +UDP transport, using the +:api:`twisted.internet.interfaces.IReactorUDP ` +reactor API. + + + DatagramProtocol ---------------- -The class where you actually implement the protocol parsing and handling will usually be descended from :api:`twisted.internet.protocol.DatagramProtocol ` or from one of its convenience children. -The ``DatagramProtocol`` class receives datagrams and can send them out over the network. -Received datagrams include the address they were sent from. -When sending datagrams the destination address must be specified. -Here is a simple example:: + +The class where you actually implement the protocol parsing and handling +will usually be descended +from :api:`twisted.internet.protocol.DatagramProtocol ` or +from one of its convenience children. The ``DatagramProtocol`` +class receives datagrams and can send them out over the network. Received +datagrams include the address they were sent from. When sending datagrams +the destination address must be specified. + + + +Here is a simple example: + + + + +.. code-block:: python + + from twisted.internet.protocol import DatagramProtocol from twisted.internet import reactor - + class Echo(DatagramProtocol): - + def datagramReceived(self, data, (host, port)): print "received %r from %s:%d" % (data, host, port) self.transport.write(data, (host, port)) - + reactor.listenUDP(9999, Echo()) reactor.run() -As you can see, the protocol is registered with the reactor. -This means it may be persisted if it's added to an application, and thus it has :api:`twisted.internet.protocol.AbstractDatagramProtocol.startProtocol ` and :api:`twisted.internet.protocol.AbstractDatagramProtocol.stopProtocol ` methods that will get called when the protocol is connected and disconnected from a UDP socket. -The protocol's ``transport`` attribute will implement the :api:`twisted.internet.interfaces.IUDPTransport ` interface. -Notice that the ``host`` argument should be an IP address, not a hostname. -If you only have the hostname use ``reactor.resolve()`` to resolve the address (see :api:`twisted.internet.interfaces.IReactorCore.resolve `). + + +As you can see, the protocol is registered with the reactor. This means +it may be persisted if it's added to an application, and thus it has +:api:`twisted.internet.protocol.AbstractDatagramProtocol.startProtocol ` +and :api:`twisted.internet.protocol.AbstractDatagramProtocol.stopProtocol ` +methods that will get called when the protocol is connected and disconnected +from a UDP socket. + + + + +The protocol's ``transport`` attribute will +implement the :api:`twisted.internet.interfaces.IUDPTransport ` interface. +Notice that the ``host`` argument should be an +IP address, not a hostname. If you only have the hostname use ``reactor.resolve()`` to resolve the address (see :api:`twisted.internet.interfaces.IReactorCore.resolve ` ). + + + Adopted Datagram Ports ---------------------- -It is also possible to add an existing ``SOCK_DGRAM`` file descriptor to the reactor using the :api:`twisted.internet.interfaces.IReactorSocket.adoptDatagramPort ` API. + + +It is also possible to add an +existing ``SOCK_DGRAM`` file descriptor to +the reactor using the :api:`twisted.internet.interfaces.IReactorSocket.adoptDatagramPort ` +API. + + + Here is a simple example: + + + + :download:`adopt_datagram_port.py ` .. literalinclude:: listings/udp/adopt_datagram_port.py .. note:: - - You must ensure that the socket is non-blocking before passing its file descriptor to :api:`twisted.internet.interfaces.IReactorSocket. adoptDatagramPort `. - - :api:`twisted.internet.interfaces.IReactorSocket. adoptDatagramPort ` cannot (`currently `_) detect the family of the adopted socket so you must ensure that you pass the correct socket family argument. - - The reactor will not shutdown the socket. - It is the responsibility of the process that created the socket to shutdown and clean up the socket when it is no longer needed. - + + + + + - You must ensure that the socket is non-blocking before + passing its file descriptor to :api:`twisted.internet.interfaces.IReactorSocket. adoptDatagramPort < adoptDatagramPort>` . + - :api:`twisted.internet.interfaces.IReactorSocket. adoptDatagramPort < adoptDatagramPort>` cannot + (`currently `_ ) + detect the family of the adopted socket so you must ensure that + you pass the correct socket family argument. + - The reactor will not shutdown the socket. It is the + responsibility of the process that created the socket to + shutdown and clean up the socket when it is no longer + needed. + + + + + + + Connected UDP ------------- -A connected UDP socket is slightly different from a standard one as it can only send and receive datagrams to/from a single address. -However this does not in any way imply a connection as datagrams may still arrive in any order and the port on the other side may have no one listening. -The benefit of the connected UDP socket is that it **may** provide notification of undelivered packages. -This depends on many factors (almost all of which are out of the control of the application) but still presents certain benefits which occasionally make it useful. -Unlike a regular UDP protocol, we do not need to specify where to send datagrams and are not told where they came from since they can only come from the address to which the socket is 'connected'. + +A connected UDP socket is slightly different from a standard one - it +can only send and receive datagrams to/from a single address, but this +does not in any way imply a connection. Datagrams may still arrive in any +order, and the port on the other side may have no one listening. The +benefit of the connected UDP socket is that it it **may** +provide notification of undelivered packages. This depends on many +factors, almost all of which are out of the control of the application, +but it still presents certain benefits which occasionally make it +useful. + + + + +Unlike a regular UDP protocol, we do not need to specify where to send +datagrams and are not told where they came from since they can only come +from the address to which the socket is 'connected'. + + + + .. code-block:: python + from twisted.internet.protocol import DatagramProtocol from twisted.internet import reactor - + class Helloer(DatagramProtocol): - + def startProtocol(self): host = "192.168.1.1" port = 1234 - + self.transport.connect(host, port) print "now we can only send to host %s port %d" % (host, port) self.transport.write("hello") # no need for address - + def datagramReceived(self, data, (host, port)): print "received %r from %s:%d" % (data, host, port) - + # Possibly invoked if there is no server listening on the # address to which we are sending. def connectionRefused(self): print "No one listening" - + # 0 means any port, we don't care in this case reactor.listenUDP(0, Helloer()) reactor.run() -Note that ``connect()``, like ``write()`` will only accept IP addresses, not unresolved hostnames. -To obtain the IP of a hostname use ``reactor.resolve()`` , e.g.:: - from twisted.internet import reactor + +Note that ``connect()`` , +like ``write()`` will only accept IP addresses, not +unresolved hostnames. To obtain the IP of a hostname +use ``reactor.resolve()`` , e.g.: + + + + + +.. code-block:: python + + + from twisted.internet import reactor + def gotIP(ip): print "IP of 'example.com' is", ip reactor.callLater(3, reactor.stop) - + reactor.resolve('example.com').addCallback(gotIP) reactor.run() -Connecting to a new address after a previous connection or making a connected port unconnected are not currently supported, but likely will be in the future. + + + +Connecting to a new address after a previous connection or making a +connected port unconnected are not currently supported, but likely will be +in the future. + + + Multicast UDP ------------- -Multicast allows a process to contact multiple hosts with a single packet, without knowing the specific IP address of any of the hosts. -This is in contrast to normal, or unicast, UDP, where each datagram has a single IP as its destination. -Multicast datagrams are sent to special multicast group addresses (in the IPv4 range 224.0.0.0 to 239.255.255.255), along with a corresponding port. -In order to receive multicast datagrams, you must join that specific group address. -However, any UDP socket can send to multicast addresses. + + +Multicast allows a process to contact multiple hosts with a single +packet, without knowing the specific IP address of any of the hosts. This +is in contrast to normal, or unicast, UDP, where each datagram has a single +IP as its destination. Multicast datagrams are sent to special multicast +group addresses (in the IPv4 range 224.0.0.0 to 239.255.255.255), along with +a corresponding port. In order to receive multicast datagrams, you must +join that specific group address. However, any UDP socket can send to +multicast addresses. + + + + :download:`MulticastServer.py ` .. literalinclude:: listings/udp/MulticastServer.py -As with UDP, with multicast there is no server/client differentiation at the protocol level. -Our server example is very simple and closely resembles a normal :api:`twisted.internet.interfaces.IReactorUDP.listenUDP ` protocol implementation. -The main difference is that instead of ``listenUDP``, :api:`twisted.internet.interfaces.IReactorMulticast.listenMulticast ` is called with the port number. -The server calls :api:`twisted.internet.interfaces.IMulticastTransport.joinGroup ` to join a multicast group. -A ``DatagramProtocol`` that is listening with multicast and has joined a group can receive multicast datagrams, but also unicast datagrams sent directly to its address. -The server in the example above sends such a unicast message in reply to the multicast message it receives from the client. -:download:`MulticastClient.py ` +As with UDP, with multicast there is no server/client differentiation +at the protocol level. Our server example is very simple and closely +resembles a normal :api:`twisted.internet.interfaces.IReactorUDP.listenUDP ` +protocol implementation. The main difference is that instead +of ``listenUDP`` , :api:`twisted.internet.interfaces.IReactorMulticast.listenMulticast ` +is called with the port number. The server calls :api:`twisted.internet.interfaces.IMulticastTransport.joinGroup ` to +join a multicast group. A ``DatagramProtocol`` +that is listening with multicast and has joined a group can receive +multicast datagrams, but also unicast datagrams sent directly to its +address. The server in the example above sends such a unicast message in +reply to the multicast message it receives from the client. -.. literalinclude:: listings/udp/MulticastClient.py -Note that a multicast socket will have a default TTL (time to live) of 1. -That is, datagrams won't traverse more than one router hop, unless a higher TTL is set with :api:`twisted.internet.interfaces.IMulticastTransport.setTTL `. -Other functionality provided by the multicast transport includes :api:`twisted.internet.interfaces.IMulticastTransport.setOutgoingInterface ` and :api:`twisted.internet.interfaces.IMulticastTransport.setLoopbackMode ` -- see :api:`twisted.internet.interfaces.IMulticastTransport ` for more information. + -Broadcast UDP -------------- -Broadcast allows a different way of contacting several unknown hosts. -Broadcasting via UDP sends a packet out to all hosts on the local network by sending to a magic broadcast address (``""``). -This broadcast is filtered by routers by default, and there are no "groups" like multicast, only different ports. +:download:`MulticastClient.py ` -Broadcast is enabled by passing ``True`` to :api:`twisted.internet.interfaces.IUDPTransport.setBroadcastAllowed ` on the port. -Checking the broadcast status can be done with :api:`twisted.internet.interfaces.IUDPTransport.getBroadcastAllowed ` on the port. +.. literalinclude:: listings/udp/MulticastClient.py + + +Note that a multicast socket will have a default TTL (time to live) of +1. That is, datagrams won't traverse more than one router hop, unless a +higher TTL is set with +:api:`twisted.internet.interfaces.IMulticastTransport.setTTL ` . Other +functionality provided by the multicast transport +includes :api:`twisted.internet.interfaces.IMulticastTransport.setOutgoingInterface ` +and :api:`twisted.internet.interfaces.IMulticastTransport.setLoopbackMode ` +-- see :api:`twisted.internet.interfaces.IMulticastTransport ` for more +information. + + -For a complete example of this feature, see :download:`udpbroadcast.py `. IPv6 ---- + + UDP sockets can also bind to IPv6 addresses to support sending and receiving datagrams over IPv6. -By passing an IPv6 address to :api:`twisted.internet.interfaces.IReactorUDP.listenUDP `'s ``interface`` argument, the reactor will start an IPv6 socket that can be used to send and receive UDP datagrams. +By passing an IPv6 address to :api:`twisted.internet.interfaces.IReactorUDP.listenUDP ` 's ``interface`` argument, +the reactor will start an IPv6 socket that can be used to send and receive UDP datagrams. + + + + + :download:`ipv6_listen.py ` .. literalinclude:: listings/udp/ipv6_listen.py + diff --git a/docs/projects/names/howto/index.rst b/docs/projects/names/howto/index.rst index aaf1f51..32e37f9 100644 --- a/docs/projects/names/howto/index.rst +++ b/docs/projects/names/howto/index.rst @@ -8,9 +8,7 @@ Developer Guides .. toctree:: :hidden: - client-tour names - custom-server - :doc:`A guided tour of twisted.names.client ` - :doc:`Using the twistd plugin ` diff --git a/docs/projects/web/howto/using-twistedweb.rst b/docs/projects/web/howto/using-twistedweb.rst index 0be5c2f..fdfc1fa 100644 --- a/docs/projects/web/howto/using-twistedweb.rst +++ b/docs/projects/web/howto/using-twistedweb.rst @@ -438,6 +438,13 @@ Twisted Web provides an abstraction of this browser-tracking behavior called the +The default session cookie name is ``TWISTED_SESSION``. It can be change by +overwriting the default implementation of +:api:`twisted.web.iweb.IRequest#getSessionCookieName +`. + + + .. image:: ../img/web-session.png diff --git a/docs/projects/web/howto/web-in-60/access-logging.rst b/docs/projects/web/howto/web-in-60/access-logging.rst deleted file mode 100644 index 9dd3bdb..0000000 --- a/docs/projects/web/howto/web-in-60/access-logging.rst +++ /dev/null @@ -1,56 +0,0 @@ - -:LastChangedDate: $LastChangedDate$ -:LastChangedRevision: $LastChangedRevision$ -:LastChangedBy: $LastChangedBy$ - -Access Logging -============== - -As long as we're on the topic of :doc:`logging `\ , this is probably a good time to mention Twisted Web's access log support. -In this example, we'll see what Twisted Web logs for each request it processes and how this can be customized. - -If you've run any of the previous examples and watched the output of ``twistd`` or read ``twistd.log`` then you've already seen some log lines like this: - - 2014-01-29 17:50:50-0500 [HTTPChannel,0,127.0.0.1] "127.0.0.1" - - [29/Jan/2014:22:50:50 +0000] "GET / HTTP/1.1" 200 2753 "-" "Mozilla/5.0 ..." - -If you focus on the latter portion of this log message you'll see something that looks like a standard "combined log format" message. -However, it's prefixed with the normal Twisted logging prefix giving a timestamp and some protocol and peer addressing information. -Much of this information is redundant since it is part of the combined log format. -:api:`twisted.web.server.Site ` lets you produce a more compact log which omits the normal Twisted logging prefix. -To take advantage of this feature all that is necessary is to tell :api:`twisted.web.server.Site ` where to write this compact log. -Do this by passing ``logPath`` to the initializer: - -.. code-block:: python - - ... - factory = Site(root, logPath=b"/tmp/access-logging-demo.log") - -Or if you want to change the logging behavior of a server you're launching with ``twistd web`` then just pass the ``--logfile`` option: - -.. code-block:: shell - - $ twistd -n web --logfile /tmp/access-logging-demo.log - -Apart from this, the rest of the server setup is the same. -Once you pass ``logPath`` or use ``--logfile`` on the command line the server will produce a log file containing lines like: - - "127.0.0.1" - - [30/Jan/2014:00:13:35 +0000] "GET / HTTP/1.1" 200 2753 "-" "Mozilla/5.0 ..." - -Any tools expecting combined log format messages should be able to work with these log files. - -:api:`twisted.web.server.Site ` also allows the log format used to be customized using its ``logFormatter`` argument. -Twisted Web comes with one alternate formatter, :api:`twisted.web.http.proxiedLogFormatter `, which is for use behind a proxy that sets the ``X-Forwarded-For`` header. -It logs the client address taken from this header rather than the network address of the client directly connected to the server. -Here's the complete code for an example that uses both these features: - -.. code-block:: python - - from twisted.web.http import proxiedLogFormatter - from twisted.web.server import Site - from twisted.web.static import File - from twisted.internet import reactor - - resource = File('/tmp') - factory = Site(resource, logPath=b"/tmp/access-logging-demo.log", logFormatter=proxiedLogFormatter) - reactor.listenTCP(8888, factory) - reactor.run() diff --git a/docs/projects/web/howto/web-in-60/index.rst b/docs/projects/web/howto/web-in-60/index.rst index 7f63232..b65a5b7 100644 --- a/docs/projects/web/howto/web-in-60/index.rst +++ b/docs/projects/web/howto/web-in-60/index.rst @@ -23,7 +23,6 @@ Twisted Web In 60 Seconds asynchronous-deferred interrupted logging-errors - access-logging wsgi http-auth session-basics @@ -54,7 +53,6 @@ here, see the :doc:`Twisted Web tutorial <../using-twistedweb>` and the API docu #. :doc:`Asynchronous responses (via Deferred) ` #. :doc:`Interrupted responses ` #. :doc:`Logging errors ` -#. :doc:`Access logging ` #. :doc:`WSGIs ` #. :doc:`HTTP authentication ` #. :doc:`Session basics ` diff --git a/twisted/internet/interfaces.py b/twisted/internet/interfaces.py index f2f0bc8..35f934e 100644 --- a/twisted/internet/interfaces.py +++ b/twisted/internet/interfaces.py @@ -2331,21 +2331,6 @@ class IUDPTransport(Interface): upon completion. """ - def setBroadcastAllowed(enabled): - """ - Set whether this port may broadcast. - - @param enabled: Whether the port may broadcast. - @type enabled: L{bool} - """ - - def getBroadcastAllowed(): - """ - Checks if broadcast is currently allowed on this port. - - @return: Whether this port may broadcast. - @rtype: L{bool} - """ class IUNIXDatagramTransport(Interface): diff --git a/twisted/internet/iocpreactor/udp.py b/twisted/internet/iocpreactor/udp.py index 54742ba..06d944f 100644 --- a/twisted/internet/iocpreactor/udp.py +++ b/twisted/internet/iocpreactor/udp.py @@ -306,28 +306,6 @@ class Port(abstract.FileHandle): return address.IPv6Address('UDP', *(addr[:2])) - def setBroadcastAllowed(self, enabled): - """ - Set whether this port may broadcast. This is disabled by default. - - @param enabled: Whether the port may broadcast. - @type enabled: L{bool} - """ - self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_BROADCAST, enabled) - - - def getBroadcastAllowed(self): - """ - Checks if broadcast is currently allowed on this port. - - @return: Whether this port may broadcast. - @rtype: L{bool} - """ - return operator.truth( - self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST)) - - class MulticastMixin: """ diff --git a/twisted/internet/test/test_udp.py b/twisted/internet/test/test_udp.py index f8e17d4..d6548f2 100644 --- a/twisted/internet/test/test_udp.py +++ b/twisted/internet/test/test_udp.py @@ -360,17 +360,6 @@ class UDPPortTestsMixin(object): error.InvalidAddressError, port.connect, 'example.invalid', 1) - def test_allowBroadcast(self): - """ - L{IListeningPort.setBroadcastAllowed} sets broadcast to be allowed - on the socket. - """ - reactor = self.buildReactor() - port = self.getListeningPort(reactor, DatagramProtocol()) - port.setBroadcastAllowed(True) - self.assertTrue(port.getBroadcastAllowed()) - - class UDPServerTestsBuilder(ReactorBuilder, UDPPortTestsMixin, DatagramTransportTestsMixin): diff --git a/twisted/internet/udp.py b/twisted/internet/udp.py index 63328a7..f2793ba 100644 --- a/twisted/internet/udp.py +++ b/twisted/internet/udp.py @@ -396,28 +396,6 @@ class Port(base.BasePort): return address.IPv6Address('UDP', *(addr[:2])) - def setBroadcastAllowed(self, enabled): - """ - Set whether this port may broadcast. This is disabled by default. - - @param enabled: Whether the port may broadcast. - @type enabled: L{bool} - """ - self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_BROADCAST, enabled) - - - def getBroadcastAllowed(self): - """ - Checks if broadcast is currently allowed on this port. - - @return: Whether this port may broadcast. - @rtype: L{bool} - """ - return operator.truth( - self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST)) - - class MulticastMixin: """ diff --git a/twisted/mail/alias.py b/twisted/mail/alias.py index c9d1807..1741b1f 100644 --- a/twisted/mail/alias.py +++ b/twisted/mail/alias.py @@ -5,9 +5,14 @@ """ -Support for aliases(5) configuration files. +Support for aliases(5) configuration files @author: Jp Calderone + +TODO:: + Monitor files for reparsing + Handle non-local alias targets + Handle maildir alias targets """ import os @@ -23,22 +28,6 @@ from zope.interface import implements, Interface def handle(result, line, filename, lineNo): - """ - Parse a line from an aliases file. - - @type result: L{dict} mapping L{bytes} to L{list} of L{bytes} - @param result: A dictionary mapping username to aliases to which - the results of parsing the line are added. - - @type line: L{bytes} - @param line: A line from an aliases file. - - @type filename: L{bytes} - @param filename: The full or relative path to the aliases file. - - @type lineNo: L{int} - @param lineNo: The position of the line within the aliases file. - """ parts = [p.strip() for p in line.split(':', 1)] if len(parts) != 2: fmt = "Invalid format on line %d of alias file %s." @@ -48,48 +37,33 @@ def handle(result, line, filename, lineNo): user, alias = parts result.setdefault(user.strip(), []).extend(map(str.strip, alias.split(','))) - - def loadAliasFile(domains, filename=None, fp=None): - """ - Load a file containing email aliases. + """Load a file containing email aliases. Lines in the file should be formatted like so:: - username: alias1, alias2, ..., aliasN + username: alias1,alias2,...,aliasN - Aliases beginning with a C{|} will be treated as programs, will be run, and + Aliases beginning with a | will be treated as programs, will be run, and the message will be written to their stdin. - Aliases beginning with a C{:} will be treated as a file containing - additional aliases for the username. - - Aliases beginning with a C{/} will be treated as the full pathname to a file - to which the message will be appended. - Aliases without a host part will be assumed to be addresses on localhost. If a username is specified multiple times, the aliases for each are joined together as if they had all been on one line. - Lines beginning with a space or a tab are continuations of the previous - line. - - Lines beginning with a C{#} are comments. - - @type domains: L{dict} mapping L{bytes} to L{IDomain} provider - @param domains: A mapping of domain name to domain object. + @type domains: C{dict} of implementor of C{IDomain} + @param domains: The domains to which these aliases will belong. - @type filename: L{bytes} or L{NoneType } - @param filename: The full or relative path to a file from which to load - aliases. If omitted, the C{fp} parameter must be specified. + @type filename: C{str} + @param filename: The filename from which to load aliases. - @type fp: file-like object or L{NoneType } - @param fp: The file from which to load aliases. If specified, - the C{filename} parameter is ignored. + @type fp: Any file-like object. + @param fp: If specified, overrides C{filename}, and aliases are read from + it. - @rtype: L{dict} mapping L{bytes} to L{AliasGroup} - @return: A mapping from username to group of aliases. + @rtype: C{dict} + @return: A dictionary mapping usernames to C{AliasGroup} objects. """ result = {} if fp is None: @@ -116,69 +90,19 @@ def loadAliasFile(domains, filename=None, fp=None): result[u] = AliasGroup(a, domains, u) return result - - class IAlias(Interface): - """ - An interface for aliases. - """ def createMessageReceiver(): - """ - Create a message receiver. - - @rtype: L{IMessage } provider - @return: A message receiver. - """ - - + pass class AliasBase: - """ - The default base class for aliases. - - @ivar domains: See L{__init__}. - - @type original: L{Address} - @ivar original: The original address being aliased. - """ def __init__(self, domains, original): - """ - @type domains: L{dict} mapping L{bytes} to L{IDomain} provider - @param domains: A mapping of domain name to domain object. - - @type original: L{bytes} - @param original: The original address being aliased. - """ self.domains = domains self.original = smtp.Address(original) - def domain(self): - """ - Return the domain associated with original address. - - @rtype: L{IDomain} provider - @return: The domain for the original address. - """ return self.domains[self.original.domain] - def resolve(self, aliasmap, memo=None): - """ - Map this alias to its ultimate destination. - - @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} - @param aliasmap: A mapping of username to alias or group of aliases. - - @type memo: L{NoneType } or L{dict} of L{AliasBase} - @param memo: A record of the aliases already considered in the - resolution process. If provided, C{memo} is modified to include - this alias. - - @rtype: L{IMessage } or L{NoneType } - @return: A message receiver for the ultimate destination or None for - an invalid destination. - """ if memo is None: memo = {} if str(self) in memo: @@ -190,64 +114,22 @@ class AliasBase: class AddressAlias(AliasBase): """ - An alias which translates one email address into another. - - @type alias : L{Address} - @ivar alias: The destination address. + The simplest alias, translating one email address into another. """ + implements(IAlias) def __init__(self, alias, *args): - """ - @type alias: L{Address}, L{User}, L{bytes} or object which can be - converted into L{bytes} - @param alias: The destination address. - - @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} - provider, (1) L{bytes} - @param args: Arguments for L{AliasBase.__init__}. - """ AliasBase.__init__(self, *args) self.alias = smtp.Address(alias) - def __str__(self): - """ - Build a string representation of this L{AddressAlias} instance. - - @rtype: L{bytes} - @return: A string containing the destination address. - """ return '
' % (self.alias,) - def createMessageReceiver(self): - """ - Create a message receiver which delivers a message to - the destination address. - - @rtype: L{IMessage } provider - @return: A message receiver. - """ return self.domain().exists(str(self.alias)) - def resolve(self, aliasmap, memo=None): - """ - Map this alias to its ultimate destination. - - @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} - @param aliasmap: A mapping of username to alias or group of aliases. - - @type memo: L{NoneType } or L{dict} of L{AliasBase} - @param memo: A record of the aliases already considered in the - resolution process. If provided, C{memo} is modified to include - this alias. - - @rtype: L{IMessage } or L{NoneType } - @return: A message receiver for the ultimate destination or None for - an invalid destination. - """ if memo is None: memo = {} if str(self) in memo: @@ -261,51 +143,17 @@ class AddressAlias(AliasBase): return aliasmap[self.alias.local].resolve(aliasmap, memo) return None - - class FileWrapper: - """ - A message receiver which delivers a message to a file. - - @type fp: file-like object - @ivar fp: A file used for temporary storage of the message. - - @type finalname: L{bytes} - @ivar finalname: The name of the file in which the message should be - stored. - """ implements(smtp.IMessage) def __init__(self, filename): - """ - @type filename: L{bytes} - @param filename: The name of the file in which the message should be - stored. - """ self.fp = tempfile.TemporaryFile() self.finalname = filename - def lineReceived(self, line): - """ - Write a received line to the temporary file. - - @type line: L{bytes} - @param line: A received line of the message. - """ self.fp.write(line + '\n') - def eomReceived(self): - """ - Handle end of message by writing the message to the file. - - @rtype: L{Deferred } which successfully results in - L{bytes} - @return: A deferred which succeeds with the name of the file to which - the message has been stored or fails if the message cannot be - saved to the file. - """ self.fp.seek(0, 0) try: f = file(self.finalname, 'a') @@ -318,105 +166,54 @@ class FileWrapper: return defer.succeed(self.finalname) - def connectionLost(self): - """ - Close the temporary file when the connection is lost. - """ self.fp.close() self.fp = None - def __str__(self): - """ - Build a string representation of this L{FileWrapper} instance. - - @rtype: L{bytes} - @return: A string containing the file name of the message. - """ return '' % (self.finalname,) - class FileAlias(AliasBase): - """ - An alias which translates an address to a file. - @ivar filename: See L{__init__}. - """ implements(IAlias) def __init__(self, filename, *args): - """ - @type filename: L{bytes} - @param filename: The name of the file in which to store the message. - - @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} - provider, (1) L{bytes} - @param args: Arguments for L{AliasBase.__init__}. - """ AliasBase.__init__(self, *args) self.filename = filename - def __str__(self): - """ - Build a string representation of this L{FileAlias} instance. - - @rtype: L{bytes} - @return: A string containing the name of the file. - """ return '' % (self.filename,) - def createMessageReceiver(self): - """ - Create a message receiver which delivers a message to the file. - - @rtype: L{FileWrapper} - @return: A message receiver which writes a message to the file. - """ return FileWrapper(self.filename) class ProcessAliasTimeout(Exception): """ - An error indicating that a timeout occurred while waiting for a process - to complete. + A timeout occurred while processing aliases. """ class MessageWrapper: """ - A message receiver which delivers a message to a child process. + A message receiver which delivers content to a child process. - @type completionTimeout: L{int} or L{float} + @type completionTimeout: C{int} or C{float} @ivar completionTimeout: The number of seconds to wait for the child process to exit before reporting the delivery as a failure. - @type _timeoutCallID: L{NoneType } or - L{IDelayedCall } provider + @type _timeoutCallID: C{NoneType} or L{IDelayedCall} @ivar _timeoutCallID: The call used to time out delivery, started when the connection to the child process is closed. - @type done: L{bool} - @ivar done: A flag indicating whether the child process has exited - (C{True}) or not (C{False}). - - @type reactor: L{IReactorTime } - provider - @ivar reactor: A reactor which will be used to schedule timeouts. - - @ivar protocol: See L{__init__}. - - @type processName: L{bytes} or L{NoneType } - @ivar processName: The process name. + @type done: C{bool} + @ivar done: Flag indicating whether the child process has exited or not. - @type completion: L{Deferred } - @ivar completion: The deferred which will be triggered by the protocol - when the child process exits. + @ivar reactor: An L{IReactorTime} provider which will be used to schedule + timeouts. """ implements(smtp.IMessage) @@ -428,17 +225,6 @@ class MessageWrapper: reactor = reactor def __init__(self, protocol, process=None, reactor=None): - """ - @type protocol: L{ProcessAliasProtocol} - @param protocol: The protocol associated with the child process. - - @type process: L{bytes} or L{NoneType } - @param process: The process name. - - @type reactor: L{NoneType } or L{IReactorTime - } provider - @param reactor: A reactor which will be used to schedule timeouts. - """ self.processName = process self.protocol = protocol self.completion = defer.Deferred() @@ -452,14 +238,6 @@ class MessageWrapper: def _processEnded(self, result): """ Record process termination and cancel the timeout call if it is active. - - @type result: L{Failure } - @param result: The reason the child process terminated. - - @rtype: L{NoneType } or - L{Failure } - @return: None, if the process end is expected, or the reason the child - process terminated, if the process end is unexpected. """ self.done = True if self._timeoutCallID is not None: @@ -474,12 +252,6 @@ class MessageWrapper: def lineReceived(self, line): - """ - Write a received line to the child process. - - @type line: L{bytes} - @param line: A received line of the message. - """ if self.done: return self.protocol.transport.write(line + '\n') @@ -487,12 +259,9 @@ class MessageWrapper: def eomReceived(self): """ - Disconnect from the child process and set up a timeout to wait for it - to exit. - - @rtype: L{Deferred } - @return: A deferred which will be called back when the child process - exits. + Disconnect from the child process, set up a timeout to wait for it to + exit, and return a Deferred which will be called back when the child + process exits. """ if not self.done: self.protocol.transport.loseConnection() @@ -505,7 +274,7 @@ class MessageWrapper: """ Handle the expiration of the timeout for the child process to exit by terminating the child process forcefully and issuing a failure to the - L{completion} deferred. + completion deferred returned by L{eomReceived}. """ self._timeoutCallID = None self.protocol.transport.signalProcess('KILL') @@ -516,38 +285,29 @@ class MessageWrapper: def connectionLost(self): - """ - Ignore notification of lost connection. - """ + # Heh heh + pass def __str__(self): - """ - Build a string representation of this L{MessageWrapper} instance. - - @rtype: L{bytes} - @return: A string containing the name of the process. - """ return '' % (self.processName,) class ProcessAliasProtocol(protocol.ProcessProtocol): """ - A process protocol which errbacks a deferred when the associated + Trivial process protocol which will callback a Deferred when the associated process ends. - @type onEnd: L{NoneType } or L{Deferred } - @ivar onEnd: If set, a deferred on which to errback when the process ends. + @ivar onEnd: If not C{None}, a L{Deferred} which will be called back with + the failure passed to C{processEnded}, when C{processEnded} is called. """ + onEnd = None def processEnded(self, reason): """ - Call an errback. - - @type reason: L{Failure } - @param reason: The reason the child process terminated. + Call back C{onEnd} if it is set. """ if self.onEnd is not None: self.onEnd.errback(reason) @@ -556,36 +316,16 @@ class ProcessAliasProtocol(protocol.ProcessProtocol): class ProcessAlias(AliasBase): """ - An alias which is handled by the execution of a program. + An alias which is handled by the execution of a particular program. - @type path: L{list} of L{bytes} - @ivar path: The arguments to pass to the process. The first string is - the executable's name. - - @type program: L{bytes} - @ivar program: The path of the program to be executed. - - @type reactor: L{IReactorTime } - and L{IReactorProcess } - provider - @ivar reactor: A reactor which will be used to create and timeout the - child process. + @ivar reactor: An L{IReactorProcess} and L{IReactorTime} provider which + will be used to create and timeout the alias child process. """ implements(IAlias) reactor = reactor def __init__(self, path, *args): - """ - @type path: L{bytes} - @param path: The command to invoke the program consisting of the path - to the executable followed by any arguments. - - @type args: 2-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} - provider, (1) L{bytes} - @param args: Arguments for L{AliasBase.__init__}. - """ - AliasBase.__init__(self, *args) self.path = path.split() self.program = self.path[0] @@ -593,48 +333,22 @@ class ProcessAlias(AliasBase): def __str__(self): """ - Build a string representation of this L{ProcessAlias} instance. - - @rtype: L{bytes} - @return: A string containing the command used to invoke the process. + Build a string representation containing the path. """ return '' % (self.path,) def spawnProcess(self, proto, program, path): """ - Spawn a process. - - This wraps the L{spawnProcess - } method on - L{reactor} so that it can be customized for test purposes. - - @type proto: L{IProcessProtocol - } provider - @param proto: An object which will be notified of all events related to - the created process. - - @type program: L{bytes} - @param program: The full path name of the file to execute. - - @type path: L{list} of L{bytes} - @param path: The arguments to pass to the process. The first string - should be the executable's name. - - @rtype: L{IProcessTransport - } provider - @return: A process transport. + Wrapper around C{reactor.spawnProcess}, to be customized for tests + purpose. """ return self.reactor.spawnProcess(proto, program, path) def createMessageReceiver(self): """ - Launch a process and create a message receiver to pass a message - to the process. - - @rtype: L{MessageWrapper} - @return: A message receiver which delivers a message to the process. + Create a message receiver by launching a process. """ p = ProcessAliasProtocol() m = MessageWrapper(p, self.program, self.reactor) @@ -645,96 +359,45 @@ class ProcessAlias(AliasBase): class MultiWrapper: """ - A message receiver which delivers a single message to multiple other - message receivers. - - @ivar objs: See L{__init__}. + Wrapper to deliver a single message to multiple recipients. """ + implements(smtp.IMessage) def __init__(self, objs): - """ - @type objs: L{list} of L{IMessage } provider - @param objs: Message receivers to which the incoming message should be - directed. - """ self.objs = objs - def lineReceived(self, line): - """ - Pass a received line to the message receivers. - - @type line: L{bytes} - @param line: A line of the message. - """ for o in self.objs: o.lineReceived(line) - def eomReceived(self): - """ - Pass the end of message along to the message receivers. - - @rtype: L{DeferredList } whose successful results - are L{bytes} or L{NoneType } - @return: A deferred list which triggers when all of the message - receivers have finished handling their end of message. - """ return defer.DeferredList([ o.eomReceived() for o in self.objs ]) - def connectionLost(self): - """ - Inform the message receivers that the connection has been lost. - """ for o in self.objs: o.connectionLost() - def __str__(self): - """ - Build a string representation of this L{MultiWrapper} instance. - - @rtype: L{bytes} - @return: A string containing a list of the message receivers. - """ return '' % (map(str, self.objs),) class AliasGroup(AliasBase): """ - An alias which points to multiple destination aliases. - - @type processAliasFactory: no-argument callable which returns - L{ProcessAlias} - @ivar processAliasFactory: A factory for process aliases. + An alias which points to more than one recipient. - @type aliases: L{list} of L{AliasBase} which implements L{IAlias} - @ivar aliases: The destination aliases. + @ivar processAliasFactory: a factory for resolving process aliases. + @type processAliasFactory: C{class} """ + implements(IAlias) processAliasFactory = ProcessAlias def __init__(self, items, *args): - """ - Create a group of aliases. - - Parse a list of alias strings and, for each, create an appropriate - alias object. - - @type items: L{list} of L{bytes} - @param items: Aliases. - - @type args: n-L{tuple} of (0) L{dict} mapping L{bytes} to L{IDomain} - provider, (1) L{bytes} - @param args: Arguments for L{AliasBase.__init__}. - """ - AliasBase.__init__(self, *args) self.aliases = [] while items: @@ -757,58 +420,20 @@ class AliasGroup(AliasBase): else: self.aliases.append(AddressAlias(addr, *args)) - def __len__(self): - """ - Return the number of aliases in the group. - - @rtype: L{int} - @return: The number of aliases in the group. - """ return len(self.aliases) - def __str__(self): - """ - Build a string representation of this L{AliasGroup} instance. - - @rtype: L{bytes} - @return: A string containing the aliases in the group. - """ return '' % (', '.join(map(str, self.aliases))) - def createMessageReceiver(self): - """ - Create a message receiver for each alias and return a message receiver - which will pass on a message to each of those. - - @rtype: L{MultiWrapper} - @return: A message receiver which passes a message on to message - receivers for each alias in the group. - """ return MultiWrapper([a.createMessageReceiver() for a in self.aliases]) - def resolve(self, aliasmap, memo=None): - """ - Map each of the aliases in the group to its ultimate destination. - - @type aliasmap: L{dict} mapping L{bytes} to L{AliasBase} - @param aliasmap: A mapping of username to alias or group of aliases. - - @type memo: L{NoneType } or L{dict} of L{AliasBase} - @param memo: A record of the aliases already considered in the - resolution process. If provided, C{memo} is modified to include - this alias. - - @rtype: L{MultiWrapper} - @return: A message receiver which passes the message on to message - receivers for the ultimate destination of each alias in the group. - """ if memo is None: memo = {} r = [] for a in self.aliases: r.append(a.resolve(aliasmap, memo)) return MultiWrapper(filter(None, r)) + diff --git a/twisted/mail/tap.py b/twisted/mail/tap.py index 677cd74..7b974ab 100644 --- a/twisted/mail/tap.py +++ b/twisted/mail/tap.py @@ -4,7 +4,7 @@ """ -Support for creating mail servers with twistd. +I am the support module for creating mail servers with twistd """ import os @@ -27,39 +27,6 @@ from twisted.application import internet class Options(usage.Options, strcred.AuthOptionMixin): - """ - An options list parser for twistd mail. - - @type synopsis: L{bytes} - @ivar synopsis: A description of options for use in the usage message. - - @type optParameters: L{list} of L{list} of (0) L{bytes}, (1) L{bytes}, - (2) L{object}, (3) L{bytes}, (4) L{NoneType } or - callable which takes L{bytes} and returns L{object} - @ivar optParameters: Information about supported parameters. See - L{Options } for details. - - @type optFlags: L{list} of L{list} of (0) L{bytes}, (1) L{bytes} or - L{NoneType }, (2) L{bytes} - @ivar optFlags: Information about supported flags. See - L{Options } for details. - - @type _protoDefaults: L{dict} mapping L{bytes} to L{int} - @ivar _protoDefaults: A mapping of default service to port. - - @type compData: L{Completions } - @ivar compData: Metadata for the shell tab completion system. - - @type longdesc: L{bytes} - @ivar longdesc: A long description of the plugin for use in the usage - message. - - @type service: L{MailService} - @ivar service: The email service. - - @type last_domain: L{IDomain} provider or L{NoneType } - @ivar last_domain: The most recently specified domain. - """ synopsis = "[options]" optParameters = [ @@ -114,7 +81,7 @@ class Options(usage.Options, strcred.AuthOptionMixin): Also starts a POP mail server which will allow a client to log in using username: joe@example.com and password: password and collect any email that - has been saved in /tmp/example.com. + has been saved in /tmp/example.com. 2. SMTP relay @@ -126,9 +93,6 @@ class Options(usage.Options, strcred.AuthOptionMixin): """ def __init__(self): - """ - Parse options and create a mail service. - """ usage.Options.__init__(self) self.service = mail.MailService() self.last_domain = None @@ -138,17 +102,7 @@ class Options(usage.Options, strcred.AuthOptionMixin): def addEndpoint(self, service, description, certificate=None): """ - Add an endpoint to a service. - - @type service: L{bytes} - @param service: A service, either C{b'smtp'} or C{b'pop3'}. - - @type description: L{bytes} - @param description: An endpoint description string or a TCP port - number. - - @type certificate: L{bytes} or L{NoneType } - @param certificate: The name of a file containing an SSL certificate. + Given a 'service' (pop3 or smtp), add an endpoint. """ self[service].append( _toEndpoint(description, certificate=certificate)) @@ -156,12 +110,11 @@ class Options(usage.Options, strcred.AuthOptionMixin): def opt_pop3(self, description): """ - Add a POP3 port listener on the specified endpoint. - - You can listen on multiple ports by specifying multiple --pop3 options. - For backwards compatibility, a bare TCP port number can be specified, - but this is deprecated. [SSL Example: ssl:8995:privateKey=mycert.pem] - [default: tcp:8110] + Add a pop3 port listener on the specified endpoint. You can listen on + multiple ports by specifying multiple --pop3 options. For backwards + compatibility, a bare TCP port number can be specified, but this is + deprecated. [SSL Example: ssl:8995:privateKey=mycert.pem] [default: + tcp:8110] """ self.addEndpoint('pop3', description) opt_p = opt_pop3 @@ -169,21 +122,18 @@ class Options(usage.Options, strcred.AuthOptionMixin): def opt_smtp(self, description): """ - Add an SMTP port listener on the specified endpoint. - - You can listen on multiple ports by specifying multiple --smtp options. - For backwards compatibility, a bare TCP port number can be specified, - but this is deprecated. [SSL Example: ssl:8465:privateKey=mycert.pem] - [default: tcp:8025] + Add an smtp port listener on the specified endpoint. You can listen on + multiple ports by specifying multiple --smtp options For backwards + compatibility, a bare TCP port number can be specified, but this is + deprecated. [SSL Example: ssl:8465:privateKey=mycert.pem] [default: + tcp:8025] """ self.addEndpoint('smtp', description) opt_s = opt_smtp def opt_default(self): - """ - Make the most recently specified domain the default domain. - """ + """Make the most recently specified domain the default domain.""" if self.last_domain: self.service.addDomain('', self.last_domain) else: @@ -192,13 +142,11 @@ class Options(usage.Options, strcred.AuthOptionMixin): def opt_maildirdbmdomain(self, domain): - """ - Generate an SMTP/POP3 virtual domain. - - This option requires an argument of the form 'NAME=PATH' where NAME is - the DNS domain name for which email will be accepted and where PATH is - a the filesystem path to a Maildir folder. - [Example: 'example.com=/tmp/example.com'] + """Generate an SMTP/POP3 virtual domain. This option requires + an argument of the form 'NAME=PATH' where NAME is the DNS + Domain Name for which email will be accepted and where PATH is + a the filesystem path to a Maildir folder. [Example: + 'example.com=/tmp/example.com'] """ try: name, path = domain.split('=') @@ -210,8 +158,7 @@ class Options(usage.Options, strcred.AuthOptionMixin): opt_d = opt_maildirdbmdomain def opt_user(self, user_pass): - """ - Add a user and password to the last specified domain. + """add a user/password to the last specified domains """ try: user, password = user_pass.split('=', 1) @@ -223,19 +170,14 @@ class Options(usage.Options, strcred.AuthOptionMixin): raise usage.UsageError("Specify a domain before specifying users") opt_u = opt_user - def opt_bounce_to_postmaster(self): - """ - Send undeliverable messages to the postmaster. + """undelivered mails are sent to the postmaster """ self.last_domain.postmaster = 1 opt_b = opt_bounce_to_postmaster - def opt_aliases(self, filename): - """ - Specify an aliases(5) file to use for the last specified domain. - """ + """Specify an aliases(5) file to use for this domain""" if self.last_domain is not None: if mail.IAliasableDomain.providedBy(self.last_domain): aliases = alias.loadAliasFile(self.service.domains, filename) @@ -254,29 +196,22 @@ class Options(usage.Options, strcred.AuthOptionMixin): raise usage.UsageError("Specify a domain before specifying aliases") opt_A = opt_aliases - def _getEndpoints(self, reactor, service): """ Return a list of endpoints for the specified service, constructing defaults if necessary. - If no endpoints were configured for the service and the protocol - was not explicitly disabled with a I{--no-*} option, a default - endpoint for the service is created. - - @type reactor: L{IReactorTCP } - provider - @param reactor: If any endpoints are created, the reactor with + @param reactor: If any endpoints are created, this is the reactor with which they are created. - @type service: L{bytes} - @param service: The type of service for which to retrieve endpoints, - either C{b'pop3'} or C{b'smtp'}. + @param service: A key into self indicating the type of service to + retrieve endpoints for. This is either C{"pop3"} or C{"smtp"}. - @rtype: L{list} of L{IStreamServerEndpoint - } provider - @return: The endpoints for the specified service as configured by the - command line parameters. + @return: A C{list} of C{IServerStreamEndpoint} providers corresponding + to the command line parameters that were specified for C{service}. + If none were and the protocol was not explicitly disabled with a + I{--no-*} option, a default endpoint for the service is created + using C{self._protoDefaults}. """ if service == 'pop3' and self['pop3s'] and len(self[service]) == 1: # The single endpoint here is the POP3S service we added in @@ -300,12 +235,6 @@ class Options(usage.Options, strcred.AuthOptionMixin): def postOptions(self): - """ - Check the validity of the specified set of options and - configure authentication. - - @raise UsageError: When the set of options is invalid. - """ from twisted.internet import reactor if self['pop3s']: @@ -343,50 +272,18 @@ class Options(usage.Options, strcred.AuthOptionMixin): class AliasUpdater: - """ - A callable object which updates the aliases for a domain from an aliases(5) - file. - - @ivar domains: See L{__init__}. - @ivar domain: See L{__init__}. - """ def __init__(self, domains, domain): - """ - @type domains: L{dict} mapping L{bytes} to L{IDomain} provider - @param domains: A mapping of domain name to domain object - - @type domain: L{IAliasableDomain} provider - @param domain: The domain to update. - """ self.domains = domains self.domain = domain - - def __call__(self, new): - """ - Update the aliases for a domain from an aliases(5) file. - - @type new: L{bytes} - @param new: The name of an aliases(5) file. - """ self.domain.setAliasGroup(alias.loadAliasFile(self.domains, new)) - def _toEndpoint(description, certificate=None): """ - Create an endpoint based on a description. - - @type description: L{bytes} - @param description: An endpoint description string or a TCP port - number. - - @type certificate: L{bytes} or L{NoneType } - @param certificate: The name of a file containing an SSL certificate. - - @rtype: L{IStreamServerEndpoint - } provider - @return: An endpoint. + Tries to guess whether a description is a bare TCP port or a endpoint. If a + bare port is specified and a certificate file is present, returns an + SSL4ServerEndpoint and otherwise returns a TCP4ServerEndpoint. """ from twisted.internet import reactor try: @@ -406,22 +303,19 @@ def _toEndpoint(description, certificate=None): return endpoints.TCP4ServerEndpoint(reactor, port) - def makeService(config): """ - Configure a service for operating a mail server. + Construct a service for operating a mail server. - The returned service may include POP3 servers, SMTP servers, or both, + The returned service may include POP3 servers or SMTP servers (or both), depending on the configuration passed in. If there are multiple servers, - they will share all of their non-network state (i.e. the same user accounts + they will share all of their non-network state (eg, the same user accounts are available on all of them). - @type config: L{Options } - @param config: Configuration options specifying which servers to include in + @param config: An L{Options} instance specifying what servers to include in the returned service and where they should keep mail data. - @rtype: L{IService } provider - @return: A service which contains the requested mail servers. + @return: An L{IService} provider which contains the requested mail servers. """ if config['esmtp']: rmType = relaymanager.SmartHostESMTPRelayingManager diff --git a/twisted/mail/topfiles/6637.doc b/twisted/mail/topfiles/6637.doc deleted file mode 100644 index 7e834c1..0000000 --- a/twisted/mail/topfiles/6637.doc +++ /dev/null @@ -1 +0,0 @@ -twisted.mail.alias now has full API documentation. diff --git a/twisted/mail/topfiles/6648.doc b/twisted/mail/topfiles/6648.doc deleted file mode 100644 index c6335b8..0000000 --- a/twisted/mail/topfiles/6648.doc +++ /dev/null @@ -1 +0,0 @@ -twisted.mail.tap now has full API documentation. diff --git a/twisted/names/test/test_rootresolve.py b/twisted/names/test/test_rootresolve.py index bb8959a..87a8ca5 100644 --- a/twisted/names/test/test_rootresolve.py +++ b/twisted/names/test/test_rootresolve.py @@ -105,21 +105,6 @@ class MemoryDatagramTransport(object): self._protocol.stopProtocol() return succeed(None) - - def setBroadcastAllowed(self, enabled): - """ - Dummy implementation to satisfy L{IUDPTransport}. - """ - pass - - - def getBroadcastAllowed(self): - """ - Dummy implementation to satisfy L{IUDPTransport}. - """ - pass - - verifyClass(IUDPTransport, MemoryDatagramTransport) diff --git a/twisted/names/topfiles/6940.misc b/twisted/names/topfiles/6940.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/pair/test/test_tuntap.py b/twisted/pair/test/test_tuntap.py index 440c040..36e0ffa 100644 --- a/twisted/pair/test/test_tuntap.py +++ b/twisted/pair/test/test_tuntap.py @@ -10,7 +10,7 @@ from __future__ import division, absolute_import import os import struct import socket -from errno import EPERM, EBADF, EINVAL, EAGAIN, EWOULDBLOCK, ENOENT, ENODEV +from errno import EPERM, EBADF, EINVAL, EAGAIN, EWOULDBLOCK, ENOENT from random import randrange from collections import deque from itertools import cycle @@ -580,9 +580,7 @@ class TestRealSystem(_RealSystem): try: return super(TestRealSystem, self).open(filename, *args, **kwargs) except OSError as e: - # The device file may simply be missing. The device file may also - # exist but be unsupported by the kernel. - if e.errno in (ENOENT, ENODEV) and filename == b"/dev/net/tun": + if ENOENT == e.errno and filename == b"/dev/net/tun": raise SkipTest("Platform lacks /dev/net/tun") raise diff --git a/twisted/pair/topfiles/6931.misc b/twisted/pair/topfiles/6931.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/python/dist.py b/twisted/python/dist.py index 9566039..07501df 100644 --- a/twisted/python/dist.py +++ b/twisted/python/dist.py @@ -432,7 +432,11 @@ def _checkCPython(sys=sys, platform=platform): """ Checks if this implementation is CPython. - This uses C{platform.python_implementation}. + On recent versions of Python, will use C{platform.python_implementation}. + On 2.5, it will try to extract the implementation from sys.subversion. On + older versions (currently the only supported older version is 2.4), checks + if C{__pypy__} is in C{sys.modules}, since PyPy is the implementation we + really care about. If it isn't, assumes CPython. This takes C{sys} and C{platform} kwargs that by default use the real modules. You shouldn't care about these -- they are for testing purposes @@ -441,7 +445,22 @@ def _checkCPython(sys=sys, platform=platform): @return: C{False} if the implementation is definitely not CPython, C{True} otherwise. """ - return platform.python_implementation() == "CPython" + try: + return platform.python_implementation() == "CPython" + except AttributeError: + # For 2.5: + try: + implementation, _, _ = sys.subversion + return implementation == "CPython" + except AttributeError: + pass + + # Are we on Pypy? + if "__pypy__" in sys.modules: + return False + + # No? Well, then we're *probably* on CPython. + return True _isCPython = _checkCPython() diff --git a/twisted/python/test/test_dist.py b/twisted/python/test/test_dist.py index d2288ee..34c6789 100644 --- a/twisted/python/test/test_dist.py +++ b/twisted/python/test/test_dist.py @@ -431,12 +431,14 @@ class FakeModule(object): fakeCPythonPlatform = FakeModule({"python_implementation": lambda: "CPython"}) fakeOtherPlatform = FakeModule({"python_implementation": lambda: "lvhpy"}) +emptyPlatform = FakeModule({}) class WithPlatformTests(TestCase): """ - Tests for L{_checkCPython} when used with a (fake) C{platform} module. + Tests for L{_checkCPython} when used with a (fake) recent C{platform} + module. """ def test_cpython(self): """ @@ -452,3 +454,73 @@ class WithPlatformTests(TestCase): says we're not running on CPython. """ self.assertFalse(dist._checkCPython(platform=fakeOtherPlatform)) + + + +fakeCPythonSys = FakeModule({"subversion": ("CPython", None, None)}) +fakeOtherSys = FakeModule({"subversion": ("lvhpy", None, None)}) + + +def _checkCPythonWithEmptyPlatform(sys): + """ + A partially applied L{_checkCPython} that uses an empty C{platform} + module (otherwise the code this test case is supposed to test won't + even be called). + """ + return dist._checkCPython(platform=emptyPlatform, sys=sys) + + + +class WithSubversionTest(TestCase): + """ + Tests for L{_checkCPython} when used with a (fake) recent (2.5+) + C{sys.subversion}. This is effectively only relevant for 2.5, since 2.6 and + beyond have L{platform.python_implementation}, which is tried first. + """ + def test_cpython(self): + """ + L{_checkCPython} returns C{True} when C{platform.python_implementation} + is unavailable and C{sys.subversion} says we're running on CPython. + """ + isCPython = _checkCPythonWithEmptyPlatform(fakeCPythonSys) + self.assertTrue(isCPython) + + + def test_other(self): + """ + L{_checkCPython} returns C{False} when C{platform.python_implementation} + is unavailable and C{sys.subversion} says we're not running on CPython. + """ + isCPython = _checkCPythonWithEmptyPlatform(fakeOtherSys) + self.assertFalse(isCPython) + + + +oldCPythonSys = FakeModule({"modules": {}}) +oldPypySys = FakeModule({"modules": {"__pypy__": None}}) + + +class OldPythonsFallbackTest(TestCase): + """ + Tests for L{_checkCPython} when used on a Python 2.4-like platform, when + neither C{platform.python_implementation} nor C{sys.subversion} is + available. + """ + def test_cpython(self): + """ + L{_checkCPython} returns C{True} when both + C{platform.python_implementation} and C{sys.subversion} are unavailable + and there is no C{__pypy__} module in C{sys.modules}. + """ + isCPython = _checkCPythonWithEmptyPlatform(oldCPythonSys) + self.assertTrue(isCPython) + + + def test_pypy(self): + """ + L{_checkCPython} returns C{False} when both + C{platform.python_implementation} and C{sys.subversion} are unavailable + and there is a C{__pypy__} module in C{sys.modules}. + """ + isCPython = _checkCPythonWithEmptyPlatform(oldPypySys) + self.assertFalse(isCPython) diff --git a/twisted/topfiles/454.feature b/twisted/topfiles/454.feature deleted file mode 100644 index 0cd53f7..0000000 --- a/twisted/topfiles/454.feature +++ /dev/null @@ -1 +0,0 @@ -Twisted's UDP implementation now supports broadcasting. \ No newline at end of file diff --git a/twisted/topfiles/6936.misc b/twisted/topfiles/6936.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/topfiles/6941.misc b/twisted/topfiles/6941.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/topfiles/6942.misc b/twisted/topfiles/6942.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/topfiles/6943.misc b/twisted/topfiles/6943.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/topfiles/6944.misc b/twisted/topfiles/6944.misc deleted file mode 100644 index e69de29..0000000 diff --git a/twisted/web/http.py b/twisted/web/http.py index 4024b0f..e9210a1 100644 --- a/twisted/web/http.py +++ b/twisted/web/http.py @@ -85,18 +85,16 @@ except ImportError: return (key.encode('charmap'), pdict) -from zope.interface import implementer, provider +from zope.interface import implementer # twisted imports -from twisted.python.compat import ( - _PY3, unicode, intToBytes, networkString, nativeString) -from twisted.python import log -from twisted.python.components import proxyForInterface +from twisted.python.compat import (_PY3, unicode, intToBytes, networkString, + nativeString) from twisted.internet import interfaces, reactor, protocol, address from twisted.internet.defer import Deferred from twisted.protocols import policies, basic +from twisted.python import log -from twisted.web.iweb import IRequest, IAccessLogFormatter from twisted.web.http_headers import _DictHeaders, Headers from twisted.web._responses import ( @@ -1825,112 +1823,6 @@ class HTTPChannel(basic.LineReceiver, policies.TimeoutMixin): request.connectionLost(reason) - -def _escape(s): - """ - Return a string like python repr, but always escaped as if surrounding - quotes were double quotes. - - @param s: The string to escape. - @type s: L{bytes} or L{unicode} - - @return: An escaped string. - @rtype: L{unicode} - """ - if not isinstance(s, bytes): - s = s.encode("ascii") - - r = repr(s) - if not isinstance(r, unicode): - r = r.decode("ascii") - if r.startswith(u"b"): - r = r[1:] - if r.startswith(u"'"): - return r[1:-1].replace(u'"', u'\\"').replace(u"\\'", u"'") - return r[1:-1] - - - -@provider(IAccessLogFormatter) -def combinedLogFormatter(timestamp, request): - """ - @return: A combined log formatted log line for the given request. - - @see: L{IAccessLogFormatter} - """ - referrer = _escape(request.getHeader(b"referer") or b"-") - agent = _escape(request.getHeader(b"user-agent") or b"-") - line = ( - u'"%(ip)s" - - %(timestamp)s "%(method)s %(uri)s %(protocol)s" ' - u'%(code)d %(length)s "%(referrer)s" "%(agent)s"' % dict( - ip=_escape(request.getClientIP() or b"-"), - timestamp=timestamp, - method=_escape(request.method), - uri=_escape(request.uri), - protocol=_escape(request.clientproto), - code=request.code, - length=request.sentLength or u"-", - referrer=referrer, - agent=agent, - )) - return line - - - -class _XForwardedForRequest(proxyForInterface(IRequest, "_request")): - """ - Add a layer on top of another request that only uses the value of an - X-Forwarded-For header as the result of C{getClientIP}. - """ - def getClientIP(self): - """ - @return: The client address (the first address) in the value of the - I{X-Forwarded-For header}. If the header is not present, return - C{b"-"}. - """ - return self._request.requestHeaders.getRawHeaders( - b"x-forwarded-for", [b"-"])[0].split(b",")[0].strip() - - # These are missing from the interface. Forward them manually. - @property - def clientproto(self): - """ - @return: The protocol version in the request. - @rtype: L{bytes} - """ - return self._request.clientproto - - @property - def code(self): - """ - @return: The response code for the request. - @rtype: L{int} - """ - return self._request.code - - @property - def sentLength(self): - """ - @return: The number of bytes sent in the response body. - @rtype: L{int} - """ - return self._request.sentLength - - - -@provider(IAccessLogFormatter) -def proxiedLogFormatter(timestamp, request): - """ - @return: A combined log formatted log line for the given request but use - the value of the I{X-Forwarded-For} header as the value for the client - IP address. - - @see: L{IAccessLogFormatter} - """ - return combinedLogFormatter(timestamp, _XForwardedForRequest(request)) - - - class HTTPFactory(protocol.ServerFactory): """ Factory for HTTP server. @@ -1942,16 +1834,6 @@ class HTTPFactory(protocol.ServerFactory): @ivar _logDateTimeCall: A delayed call for the next update to the cached log datetime string. @type _logDateTimeCall: L{IDelayedCall} provided - - @ivar _logFormatter: See the C{logFormatter} parameter to L{__init__} - - @ivar _nativeize: A flag that indicates whether the log file being written - to wants native strings (C{True}) or bytes (C{False}). This is only to - support writing to L{twisted.python.log} which, unfortunately, works - with native strings. - - @ivar _reactor: An L{IReactorTime} provider used to compute logging - timestamps. """ protocol = HTTPChannel @@ -1960,21 +1842,11 @@ class HTTPFactory(protocol.ServerFactory): timeOut = 60 * 60 * 12 - _reactor = reactor - - def __init__(self, logPath=None, timeout=60*60*12, logFormatter=None): - """ - @param logFormatter: An object to format requests into log lines for - the access log. - @type logFormatter: L{IAccessLogFormatter} provider - """ + def __init__(self, logPath=None, timeout=60*60*12): if logPath is not None: logPath = os.path.abspath(logPath) self.logPath = logPath self.timeOut = timeout - if logFormatter is None: - logFormatter = combinedLogFormatter - self._logFormatter = logFormatter # For storing the cached log datetime and the callback to update it self._logDateTime = None @@ -1985,8 +1857,8 @@ class HTTPFactory(protocol.ServerFactory): """ Update log datetime periodically, so we aren't always recalculating it. """ - self._logDateTime = datetimeToLogString(self._reactor.seconds()) - self._logDateTimeCall = self._reactor.callLater(1, self._updateLogDateTime) + self._logDateTime = datetimeToLogString() + self._logDateTimeCall = reactor.callLater(1, self._updateLogDateTime) def buildProtocol(self, addr): @@ -2005,10 +1877,8 @@ class HTTPFactory(protocol.ServerFactory): self._updateLogDateTime() if self.logPath: - self._nativeize = False self.logFile = self._openLogFile(self.logPath) else: - self._nativeize = True self.logFile = log.logfile @@ -2027,25 +1897,35 @@ class HTTPFactory(protocol.ServerFactory): """ Override in subclasses, e.g. to use twisted.python.logfile. """ - f = open(path, "ab", 1) + f = open(path, "a", 1) return f + def _escape(self, s): + # pain in the ass. Return a string like python repr, but always + # escaped as if surrounding quotes were "". + try: + s = nativeString(s) + except UnicodeError: + pass + r = repr(s) + if r[0] == "'": + return r[1:-1].replace('"', '\\"').replace("\\'", "'") + return r[1:-1] def log(self, request): """ - Write a line representing C{request} to the access log file. - - @param request: The request object about which to log. - @type request: L{Request} + Log a request's result to the logfile, by default in combined log format. """ - try: - logFile = self.logFile - except AttributeError: - pass - else: - line = self._logFormatter(self._logDateTime, request) + u"\n" - if self._nativeize: - line = nativeString(line) - else: - line = line.encode("utf-8") - logFile.write(line) + if hasattr(self, "logFile"): + line = '%s - - %s "%s" %d %s "%s" "%s"\n' % ( + request.getClientIP(), + # request.getUser() or "-", # the remote user is almost never important + self._logDateTime, + '%s %s %s' % (self._escape(request.method), + self._escape(request.uri), + self._escape(request.clientproto)), + request.code, + request.sentLength or "-", + self._escape(request.getHeader("referer") or "-"), + self._escape(request.getHeader("user-agent") or "-")) + self.logFile.write(line) diff --git a/twisted/web/iweb.py b/twisted/web/iweb.py index 11ea922..83d6d1f 100644 --- a/twisted/web/iweb.py +++ b/twisted/web/iweb.py @@ -184,6 +184,14 @@ class IRequest(Interface): """ + def getSessionCookieName(): + """ + Return the name of the cookie used for storing the session id. + + @return: The C{str} name of the session cookie. + """ + + def URLPath(): """ @return: A L{URLPath} instance which identifies the URL for which this @@ -326,28 +334,6 @@ class IRequest(Interface): -class IAccessLogFormatter(Interface): - """ - An object which can represent an HTTP request as a line of text for - inclusion in an access log file. - """ - def __call__(timestamp, request): - """ - Generate a line for the access log. - - @param timestamp: The time at which the request was completed in the - standard format for access logs. - @type timestamp: L{unicode} - - @param request: The request object about which to log. - @type request: L{twisted.web.server.Request} - - @return: One line describing the request without a trailing newline. - @rtype: L{unicode} - """ - - - class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of diff --git a/twisted/web/server.py b/twisted/web/server.py index a71eda4..1c5b62c 100644 --- a/twisted/web/server.py +++ b/twisted/web/server.py @@ -101,9 +101,12 @@ class Request(Copyable, http.Request, components.Componentized): @ivar defaultContentType: A C{bytes} giving the default I{Content-Type} value to send in responses if no other value is set. C{None} disables the default. + @ivar sessionCookieBaseName: A C{bytes} giving the base name when creating + new session cookies. """ defaultContentType = b"text/html" + sessionCookieBaseName = b'TWISTED_SESSION' site = None appRootURL = None @@ -384,7 +387,7 @@ class Request(Copyable, http.Request, components.Componentized): def getSession(self, sessionInterface = None): # Session management if not self.session: - cookiename = b"_".join([b'TWISTED_SESSION'] + self.sitepath) + cookiename = self.getSessionCookieName() sessionCookie = self.getCookie(cookiename) if sessionCookie: try: @@ -400,6 +403,12 @@ class Request(Copyable, http.Request, components.Componentized): return self.session.getComponent(sessionInterface) return self.session + def getSessionCookieName(self): + """ + See: L{iweb.IRequest.getSessionCookieName} + """ + return b"_".join([self.sessionCookieBaseName] + self.sitepath) + def _prePathURL(self, prepath): port = self.getHost().port if self.isSecure(): @@ -617,16 +626,11 @@ class Site(http.HTTPFactory): sessionFactory = Session sessionCheckTime = 1800 - def __init__(self, resource, *args, **kwargs): + def __init__(self, resource, logPath=None, timeout=60*60*12): """ - @param resource: The root of the resource hierarchy. All request - traversal for requests received by this factory will begin at this - resource. - @type resource: L{IResource} provider - - @see: L{twisted.web.http.HTTPFactory.__init__} + Initialize. """ - http.HTTPFactory.__init__(self, *args, **kwargs) + http.HTTPFactory.__init__(self, logPath=logPath, timeout=timeout) self.sessions = {} self.resource = resource diff --git a/twisted/web/test/requesthelper.py b/twisted/web/test/requesthelper.py index 14d776f..ebe24bc 100644 --- a/twisted/web/test/requesthelper.py +++ b/twisted/web/test/requesthelper.py @@ -114,7 +114,6 @@ class DummyRequest(object): self.protoSession = session or Session(0, self) self.args = {} self.outgoingHeaders = {} - self.requestHeaders = Headers() self.responseHeaders = Headers() self.responseCode = None self.headers = {} diff --git a/twisted/web/test/test_web.py b/twisted/web/test/test_web.py index 3c80595..a42a398 100644 --- a/twisted/web/test/test_web.py +++ b/twisted/web/test/test_web.py @@ -5,20 +5,20 @@ Tests for various parts of L{twisted.web}. """ -import os import zlib -from zope.interface import implementer +from zope.interface import Attribute, implementer, Interface from zope.interface.verify import verifyObject -from twisted.python.compat import _PY3, networkString -from twisted.python.filepath import FilePath +from twisted.python.compat import (_PY3, networkString, + NativeStringIO as StringIO) from twisted.trial import unittest from twisted.internet import reactor from twisted.internet.address import IPv4Address -from twisted.internet.task import Clock from twisted.web import server, resource +from twisted.internet import task from twisted.web import iweb, http, error +from twisted.python import components, log from twisted.web.test.requesthelper import DummyChannel, DummyRequest @@ -94,7 +94,7 @@ class SessionTest(unittest.TestCase): Create a site with one active session using a deterministic, easily controlled clock. """ - self.clock = Clock() + self.clock = task.Clock() self.uid = b'unique' self.site = server.Site(resource.Resource()) self.session = server.Session(self.site, self.uid, self.clock) @@ -218,8 +218,7 @@ class ConditionalTest(unittest.TestCase): self.resrc.putChild(b'', self.resrc) self.resrc.putChild(b'with-content-type', SimpleResource(b'image/jpeg')) self.site = server.Site(self.resrc) - self.site.startFactory() - self.addCleanup(self.site.stopFactory) + self.site.logFile = log.logfile # HELLLLLLLLLLP! This harness is Very Ugly. self.channel = self.site.buildProtocol(None) @@ -468,6 +467,116 @@ class RequestTests(unittest.TestCase): self.assertEqual(request.prePathURL(), b'http://example.com/foo%2Fbar') + class DummySession(server.Session): + """ + A session to help with testing. + """ + def __init__(self, site=None, uid=0): + server.Session.__init__( + self, site=site, uid=uid, reactor=task.Clock()) + + def touch(self): + self._reactor.advance(1) + return server.Session.touch(self) + + + def test_getSession(self): + """ + When a session already exists, it will return the same session and + update its modification. + """ + channel = DummyChannel() + request = server.Request(channel, 1) + session = self.DummySession() + initial_time = session.lastModified + request.session = session + + result = request.getSession() + + self.assertIs(session, result) + self.assertGreater(result.lastModified, initial_time) + + + class ISessionObject(Interface): + """ + A simple interface for testing session components. + """ + value = Attribute("A marker value for this component.") + + + @implementer(ISessionObject) + class SessionObject(object): + """ + A simple component. + """ + def __init__(self, session): + self.value = 42 + + + def test_getSessionComponent(self): + """ + When sessionInterface is provided it will return the + C{sessionInterface} component associated with this session. + """ + # Register adapter for this test and remove it once test is done. + components.registerAdapter( + self.SessionObject, server.Session, self.ISessionObject) + # Un-registration is done by registering None. + self.addCleanup( + components.getRegistry().register, + [self.ISessionObject], server.Session, '', None) + channel = DummyChannel() + request = server.Request(channel, 1) + session = self.DummySession() + request.session = session + + result = request.getSession(sessionInterface=self.ISessionObject) + + self.assertIsInstance(result, self.SessionObject) + self.assertEqual(42, result.value) + + + def test_getSessionNonExistent(self): + """ + When request (or the site associated with this request) has no + previous session, a new one is created using the name provided by + `getSessionCookieName` as cookie is set to inform the web client + about session id. + """ + site = server.Site(resource.Resource()) + site.sessionFactory = self.DummySession + channel = DummyChannel() + channel.site = site + request = server.Request(channel, 1) + request.sitepath = [] + request.site = site + + result = request.getSession() + + session_raw_cookie = '%s=%s; Path=/' % ( + request.getSessionCookieName(), result.uid,) + self.assertEqual(session_raw_cookie, request.cookies[0]) + + # Getting the session again, will return the same object. + self.assertIs(result, request.getSession()) + + + def test_getSessionCookieName(self): + """ + Default implementation returns the name based on + I{Request.sessionCookieBaseName}. + """ + baseName = b'CUSTOM_NAME' + channel = DummyChannel() + request = server.Request(channel, 1) + request.sitepath = [] + request.site = channel.site + request.sessionCookieBaseName = baseName + + name = request.getSessionCookieName() + + self.assertEqual(name, baseName) + class GzipEncoderTests(unittest.TestCase): @@ -837,301 +946,70 @@ class DummyRequestForLogTest(DummyRequest): -class AccessLogTestsMixin(object): - """ - A mixin for L{TestCase} subclasses defining tests that apply to - L{HTTPFactory} and its subclasses. - """ - def factory(self, *args, **kwargs): - """ - Get the factory class to apply logging tests to. - - Subclasses must override this method. - """ - raise NotImplementedError("Subclass failed to override factory") - - - def test_combinedLogFormat(self): - """ - The factory's C{log} method writes a I{combined log format} line to the - factory's log file. - """ - reactor = Clock() - # Set the clock to an arbitrary point in time. It doesn't matter when - # as long as it corresponds to the timestamp in the string literal in - # the assertion below. - reactor.advance(1234567890) - - logPath = self.mktemp() - factory = self.factory(logPath=logPath) - factory._reactor = reactor - factory.startFactory() - - try: - factory.log(DummyRequestForLogTest(factory)) - finally: - factory.stopFactory() - - self.assertEqual( - # Client IP - b'"1.2.3.4" ' - # Some blanks we never fill in - b'- - ' - # The current time (circa 1234567890) - b'[13/Feb/2009:23:31:30 +0000] ' - # Method, URI, version - b'"GET /dummy HTTP/1.0" ' - # Response code - b'123 ' - # Response length - b'- ' - # Value of the "Referer" header. Probably incorrectly quoted. - b'"-" ' - # Value pf the "User-Agent" header. Probably incorrectly quoted. - b'"-"' + self.linesep, - FilePath(logPath).getContent()) - - - def test_logFormatOverride(self): - """ - If the factory is initialized with a custom log formatter then that - formatter is used to generate lines for the log file. - """ - def notVeryGoodFormatter(timestamp, request): - return u"this is a bad log format" - - reactor = Clock() - reactor.advance(1234567890) - - logPath = self.mktemp() - factory = self.factory( - logPath=logPath, logFormatter=notVeryGoodFormatter) - factory._reactor = reactor - factory.startFactory() - try: - factory.log(DummyRequestForLogTest(factory)) - finally: - factory.stopFactory() - - self.assertEqual( - # self.linesep is a sad thing. - # https://twistedmatrix.com/trac/ticket/6938 - b"this is a bad log format" + self.linesep, - FilePath(logPath).getContent()) - - - -class HTTPFactoryAccessLogTests(AccessLogTestsMixin, unittest.TestCase): - """ - Tests for L{http.HTTPFactory.log}. - """ - factory = http.HTTPFactory - linesep = b"\n" - - - -class SiteAccessLogTests(AccessLogTestsMixin, unittest.TestCase): - """ - Tests for L{server.Site.log}. - """ - if _PY3: - skip = "Site not ported to Python 3 yet." - - linesep = os.linesep - - def factory(self, *args, **kwargs): - return server.Site(resource.Resource(), *args, **kwargs) - - - -class CombinedLogFormatterTests(unittest.TestCase): - """ - Tests for L{twisted.web.http.combinedLogFormatter}. - """ - def test_interface(self): - """ - L{combinedLogFormatter} provides L{IAccessLogFormatter}. - """ - self.assertTrue(verifyObject( - iweb.IAccessLogFormatter, http.combinedLogFormatter)) - - - def test_nonASCII(self): - """ - Bytes in fields of the request which are not part of ASCII are escaped - in the result. - """ - reactor = Clock() - reactor.advance(1234567890) - - timestamp = http.datetimeToLogString(reactor.seconds()) - request = DummyRequestForLogTest(http.HTTPFactory()) - request.client = IPv4Address("TCP", b"evil x-forwarded-for \x80", 12345) - request.method = b"POS\x81" - request.protocol = b"HTTP/1.\x82" - request.headers[b"referer"] = b"evil \x83" - request.headers[b"user-agent"] = b"evil \x84" - - line = http.combinedLogFormatter(timestamp, request) - self.assertEqual( - u'"evil x-forwarded-for \\x80" - - [13/Feb/2009:23:31:30 +0000] ' - u'"POS\\x81 /dummy HTTP/1.0" 123 - "evil \\x83" "evil \\x84"', - line) - - - -class ProxiedLogFormatterTests(unittest.TestCase): - """ - Tests for L{twisted.web.http.proxiedLogFormatter}. - """ - def test_interface(self): - """ - L{proxiedLogFormatter} provides L{IAccessLogFormatter}. - """ - self.assertTrue(verifyObject( - iweb.IAccessLogFormatter, http.proxiedLogFormatter)) - - - def _xforwardedforTest(self, header): - """ - Assert that a request with the given value in its I{X-Forwarded-For} - header is logged by L{proxiedLogFormatter} the same way it would have - been logged by L{combinedLogFormatter} but with 172.16.1.2 as the - client address instead of the normal value. - - @param header: An I{X-Forwarded-For} header with left-most address of - 172.16.1.2. - """ - reactor = Clock() - reactor.advance(1234567890) - - timestamp = http.datetimeToLogString(reactor.seconds()) - request = DummyRequestForLogTest(http.HTTPFactory()) - expected = http.combinedLogFormatter(timestamp, request).replace( - u"1.2.3.4", u"172.16.1.2") - request.requestHeaders.setRawHeaders(b"x-forwarded-for", [header]) - line = http.proxiedLogFormatter(timestamp, request) - - self.assertEqual(expected, line) - - - def test_xforwardedfor(self): - """ - L{proxiedLogFormatter} logs the value of the I{X-Forwarded-For} header - in place of the client address field. - """ - self._xforwardedforTest(b"172.16.1.2, 10.0.0.3, 192.168.1.4") - - - def test_extraForwardedSpaces(self): - """ - Any extra spaces around the address in the I{X-Forwarded-For} header - are stripped and not included in the log string. - """ - self._xforwardedforTest(b" 172.16.1.2 , 10.0.0.3, 192.168.1.4") - - - class TestLogEscaping(unittest.TestCase): def setUp(self): - self.logPath = self.mktemp() - self.site = http.HTTPFactory(self.logPath) - self.site.startFactory() + self.site = http.HTTPFactory() + self.site.logFile = StringIO() self.request = DummyRequestForLogTest(self.site, False) - - def assertLogs(self, line): - """ - Assert that if C{self.request} is logged using C{self.site} then - C{line} is written to the site's access log file. - - @param line: The expected line. - @type line: L{bytes} - - @raise self.failureException: If the log file contains something other - than the expected line. - """ - try: - self.site.log(self.request) - finally: - self.site.stopFactory() - logged = FilePath(self.logPath).getContent() - self.assertEqual(line, logged) - - - def test_simple(self): - """ - A I{GET} request is logged with no extra escapes. - """ + def testSimple(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"GET /dummy HTTP/1.0" 123 - "-" "-"\n') - + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "GET /dummy HTTP/1.0" 123 - "-" "-"\n') - def test_methodQuote(self): - """ - If the HTTP request method includes a quote, the quote is escaped. - """ + def testMethodQuote(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.request.method = b'G"T' - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"G\\"T /dummy HTTP/1.0" 123 - "-" "-"\n') - + self.request.method = 'G"T' + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "G\\"T /dummy HTTP/1.0" 123 - "-" "-"\n') - def test_requestQuote(self): - """ - If the HTTP request path includes a quote, the quote is escaped. - """ + def testRequestQuote(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.request.uri = b'/dummy"withquote' - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"GET /dummy\\"withquote HTTP/1.0" 123 - "-" "-"\n') - + self.request.uri='/dummy"withquote' + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "GET /dummy\\"withquote HTTP/1.0" 123 - "-" "-"\n') - def test_protoQuote(self): - """ - If the HTTP request version includes a quote, the quote is escaped. - """ + def testProtoQuote(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.request.clientproto = b'HT"P/1.0' - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"GET /dummy HT\\"P/1.0" 123 - "-" "-"\n') - + self.request.clientproto='HT"P/1.0' + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "GET /dummy HT\\"P/1.0" 123 - "-" "-"\n') - def test_refererQuote(self): - """ - If the value of the I{Referer} header contains a quote, the quote is - escaped. - """ + def testRefererQuote(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.request.headers[b'referer'] = ( - b'http://malicious" ".website.invalid') - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"GET /dummy HTTP/1.0" 123 - ' - b'"http://malicious\\" \\".website.invalid" "-"\n') - + self.request.headers['referer'] = 'http://malicious" ".website.invalid' + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "GET /dummy HTTP/1.0" 123 - "http://malicious\\" \\".website.invalid" "-"\n') - def test_userAgentQuote(self): - """ - If the value of the I{User-Agent} header contains a quote, the quote is - escaped. - """ + def testUserAgentQuote(self): self.site._logDateTime = "[%02d/%3s/%4d:%02d:%02d:%02d +0000]" % ( 25, 'Oct', 2004, 12, 31, 59) - self.request.headers[b'user-agent'] = b'Malicious Web" Evil' - self.assertLogs( - b'"1.2.3.4" - - [25/Oct/2004:12:31:59 +0000] ' - b'"GET /dummy HTTP/1.0" 123 - "-" "Malicious Web\\" Evil"\n') + self.request.headers['user-agent'] = 'Malicious Web" Evil' + self.site.log(self.request) + self.site.logFile.seek(0) + self.assertEqual( + self.site.logFile.read(), + '1.2.3.4 - - [25/Oct/2004:12:31:59 +0000] "GET /dummy HTTP/1.0" 123 - "-" "Malicious Web\\" Evil"\n') diff --git a/twisted/web/topfiles/1468.feature b/twisted/web/topfiles/1468.feature deleted file mode 100644 index 7131e41..0000000 --- a/twisted/web/topfiles/1468.feature +++ /dev/null @@ -1 +0,0 @@ -twisted.web.http.proxiedLogFormatter can now be used with twisted.web.http.HTTPFactory (and subclasses) to record X-Forwarded-For values to the access log when the HTTP server is deployed behind a reverse proxy. diff --git a/twisted/web/topfiles/6933.feature b/twisted/web/topfiles/6933.feature new file mode 100644 index 0000000..98fd006 --- /dev/null +++ b/twisted/web/topfiles/6933.feature @@ -0,0 +1 @@ +twisted.web.server.Request allows specifying a custom name for the session cookie \ No newline at end of file