CGI gzip compression module

Simple CGI output module. Uses gzip compression on the output stream if the client accepts it.

Last modified
Lines 269
Indexable Yes

Parent directory Download CGIread sitemap Main page

Quick links: done init output overload_test write_b write_body write_h write_head

  1. #!/usr/bin/python
  2. import os
  3. import sys
  4. import subprocess
  5. # 2020-09-17    Somehow broke gzip compression
  6. #               Can't get subprocess.Popen to work anymore
  7. # 2020-09-17    Increased max load average from 3.5 to 6
  8. '''
  9. compressout
  10. ===========
  11. Simple CGI output compression module.
  12. Not all webservers support compressing the output stream of CGI scripts.
  13. This script determines whether or not a client accept gzip compression and
  14. compresses the output stream of the script.
  15. NOTICE: The `cgitb` module will write to stdout if the script crashes,
  16. you should use a browser that does not accept gzip, when you are
  17. testing your scripts.
  18. NOTICE: This module uses two global variable: `use_gzip` and `body`.
  19. It's supposed to be used like an object, but rather than using a class,
  20. this is imported "pre-created" or so to say.
  21. NOTICE: There are two constants:
  22.   `max_load_avg_1min`:
  23.     This is the maximum allowed load average under one minute.
  24.     If the one minute load average exceeds this value, compressout will
  25.     return a 503 response to the client and abort the process.
  26.   `http503_body`:
  27.     A plain text error message to send if the server is overloaded.
  28. You should modify these constants to fit your own needs.
  29. Example / TL;DR
  30. ===============
  31. import compressout
  32. compressout.init()
  33. compressout.write_head('Content-Type: text/plain\r\n')
  34. compressout.write_head('Foo: test\r\nBar: test\r\n')
  35. # Blank line required for terminating the HEAD section
  36. compressout.write_head('\r\n')
  37. compressout.write_body('Hello world!\n')
  38. compressout.write_body('Bye.\n')
  39. compressout.done()
  40. Functions
  41. =========
  42. init(write_headers=True)
  43. ------------------------
  44.     Initialize the module.  This function will detect if the client
  45.     supports gzip.
  46.     If `write_headers` is True, the function writes a
  47.     'Vary: Content-Encoding' header and (if gzip is used) a
  48.     'Content-Encdoing: gzip' header.
  49. write_head(s) and write_h(s)
  50. ----------------------------
  51.     
  52.     This function is used to print all HTTP headers **and the blank line
  53.     separating the head from the body**.
  54.     Write `s` to standard output, will never go through gzip.
  55. write_body(s) and write_b(s)
  56. ----------------------------
  57.     Write part of body.
  58.     NOTICE: You need to have printed the blank line after the headers
  59.     with the `write_h` (or `write_head`) fuction.
  60.     
  61.     If gzip is supported by the client
  62.     ----------------------------------
  63.     
  64.         `s` will be appended to a local buffer which the `done` function
  65.         will compress and print.
  66.     
  67.     If gzip is not supported
  68.     ------------------------
  69.     
  70.         `s` will go straight to stdout. The `done` function won't do
  71.         anything.
  72. done()
  73. ------
  74.     Done writing output.
  75.     This function will invoke gzip.
  76. Dos and don'ts
  77. ==============
  78.     * Try to call `init` and `done` at convenient locations like on the
  79.       "outside" of a main function, i.e. don't repeat yourself by calling
  80.       these two functions everywhere in your code.
  81.     * Never call `write_head` after any call to `write_body`.
  82.     * Always call `done` when your done.
  83.     * Use only compressout to write output, otherwise you'll have a mess.
  84.     * NOTICE: The `cgitb` module will write to stdout if the script
  85.       crashes, you should use a browser that does not accept gzip,
  86.       when you are testing your scripts.  Eg, lwp-request.
  87.       `GET http://example.com/ | less` is excellent for debuggin.
  88. '''
  89. ### GLOBALS ###
  90. use_gzip = None     # Whether or not to compress the body
  91. body = ''           # The body is stored here if it is to be compressed
  92. ### END GLOBALS ###
  93. ### CONSTANTS -- Configure for your own needs ###
  94. # If the load average of the last one minute exceeds the hard coded value,
  95. # this script will return a 503 response and abort the process.
  96. max_load_avg_1min = 6       # 3.5
  97. http503_body = '''
  98. Service temporarily unavailable!
  99. Wait at least two minutes before trying again.
  100. Re-attempting prematurely may result in banning your IP address.
  101. -- END --
  102. '''
  103. #               #############################################
  104. if sys.version_info[0] > 2:
  105.     def output(s):
  106.         if isinstance(s, str):
  107.             sys.stdout.buffer.write(s.encode('utf-8'))
  108.         elif isinstance(s, bytes):
  109.             sys.stdout.buffer.write(s)
  110.         else:
  111.             raise TypeError("Unsupported datatype")
  112.     flush = sys.stdout.buffer.flush
  113. else:
  114.     output = sys.stdout.write
  115.     flush = sys.stdout.flush
  116. def overload_test(too_late=False):
  117.     '''
  118.     '''
  119.     if os.getloadavg()[0] > max_load_avg_1min:
  120.         if not too_late:
  121.             output('Status: 503\n')
  122.             output('Content-Type: text/plain\n')
  123.             output('Retry-After: 90\n')
  124.             output(http503_body)
  125.             flush()
  126.         os.abort()
  127.             
  128. def init(write_headers=True):
  129.     '''
  130.     Initialize the module.  This function will detect if the client
  131.     support gzip.
  132.     If `write_headers`, write a 'Vary' and (if used)
  133.     'Content-Encoding' header.
  134.     '''
  135.     
  136.     global use_gzip
  137.     global body
  138.     
  139.     # This is the only place where sending a 503 message will work.
  140.     # write_h:
  141.     #   - Message body may need to be compressed.
  142.     #   - Possibility of conflicting Status headers.
  143.     # write_b:
  144.     #   - Message body may need to be compressed.
  145.     #   - Message body may be application/xhtml+xml
  146.     # done:
  147.     #   - Message body needs to be compressed if `use_gzip`.
  148.     #   - Body has already been written if not `use_gzip`.
  149.     overload_test(too_late=False)
  150.         
  151.     use_gzip = 'gzip' in os.getenv('HTTP_ACCEPT_ENCODING', '')
  152.     body = ''
  153.     use_gzip = False
  154.     
  155.     if write_headers:
  156.         output('Vary: Accept-Encoding\n')
  157.         if use_gzip:
  158.             output('Content-Encoding: gzip\n')
  159. def write_head(s):
  160.     write_h(s)
  161. def write_h(s):
  162.     '''
  163.     Write part of header.
  164.     Write `s` to standard output, will never go through gzip.
  165.     '''
  166.     overload_test(too_late=True)
  167.     output(s)
  168. def write_body(s):
  169.     write_b(s)
  170. def write_b(s):
  171.     '''
  172.     Write part of body.
  173.     
  174.     gzip is supported by the client
  175.     -------------------------------
  176.     
  177.         `s` will be appended to a local buffer
  178.         which `done` will compress and print.
  179.     
  180.     gzip is not supported
  181.     ---------------------
  182.     
  183.         `s` will go straight to stdout.
  184.     '''
  185.     
  186.     global body
  187.     overload_test(too_late=True)
  188.     
  189.     if use_gzip:
  190.         body += s
  191.     else:
  192.         output(s)
  193. def done():
  194.     '''
  195.     Done writing output.
  196.     This function will invoke gzip.
  197.     '''
  198.     
  199.     overload_test(too_late=True)
  200.     if use_gzip:
  201.         gzip = subprocess.Popen(
  202.             ['gzip'],
  203.             stdin=subprocess.PIPE,
  204.             stdout=subprocess.PIPE,
  205.         )
  206.         if sys.version_info[0] > 2:
  207.             body = body.encode('utf-8')
  208.             sys.stderr.write('Body encoded\n')
  209.         sys.stderr.write('Just before communicate\n')
  210.         gzip_stdout = gzip.communicate(body)[0]
  211.         sys.stderr.write('Just after communicate\n')
  212.         #gzip_stdout = things[0]
  213.         #sys.stderr.write('After extracting data\n')
  214.         #sys.stderr.write(gzip_stderr)
  215.         output(gzip_stdout)
  216.         sys.stderr.write('done() complete\n')