static/downloads/test2.cgi - vcl-site - Git at Google

 #!/usr/local/bin/python2.7
 # -*- python -*-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
 #
 #
 # This is a Python CGI script that uses EZT to produce templated
 # mirror content and GeoIP to choose the appropriate mirror
 #

 import sys
 import os
 import cgi
 import stat
 import random
 import time
 import traceback
 import cStringIO

 # Should be installed in the main system library
 import GeoIP

 # Insert this directory into PATH so that we can import ezt
 this_dir = os.path.dirname(__file__) or '.'
 sys.path.insert(0, this_dir)

 import ezt


 # Configurable stuff
 MIRRORS_LIST = "/x1/www/www.apache.org/mirrors/mirrors.list"
 DEFAULT_REGION = "us"
 DEFAULT_TEMPLATE = "/x1/www/www.apache.org/dyn/closer.html"
 DEFAULT_LOCATION = "http://www.apache.org/dyn/closer.cgi"


 def get_region(environ):
   """Use GeoIP to find the client's country, falling back to
      DEFAULT_REGION on failure."""
   try:
     remote_ip = environ['REMOTE_ADDR'];
     gi = GeoIP.new(GeoIP.GEOIP_STANDARD)
     region = gi.country_code_by_addr(remote_ip).lower().strip()
     if region == 'gb':
       return 'uk'
     else:
       return region
   except:
     ### should we log an error here? absorbing without reporting is
     ### generally bad form.
     return DEFAULT_REGION


 def parse_mirrors(filename, country, preferred, mingood):
   """Parse the mirror database to find the best mirrors for a client.

   The Format of the mirror database is (last two fields are optional):
   ftp au  ftp://ftp.planetmirror.com/pub/apache/dist/ 1117724635 http://example.com/logo.gif http://example.com/ """

   output = { 'http'   : [ ],
              'ftp'    : [ ],
              'backup' : [ ],
              'preferred' : None,
              'logo' : None,
              'link' : None,
              }

   # Read the mirror database and put it in a list of lists
   # skip empty lines and comment
   mirrors = [line.split() for line in open(filename).readlines()
              if line.strip() and not line.startswith('#')]

   mirrors.append(['http', 'us', 'http://archive.apache.org/dist/', '9999999999'])

   # Add trailing slashes where missing.  Otherwise,
   # strcat("http://www.mirror.org", pathinfo=".foo.evil") would link to
   # http://www.mirror.org.foo.evil
   for mir in mirrors:
     if not mir[2].endswith('/'):
       mir[2] += '/'

   # grab the backup mirrors
   backupmirrors = [mir for mir in mirrors if mir[1] == 'Backup']

   # Grab the mirrors for the requested country or, failing that,
   # from the default region (us)
   for region in (country, DEFAULT_REGION):
     countrymirrors = [mir for mir in mirrors if mir[1] == region]
     random.shuffle(countrymirrors)
     goodmirror = None
     for mir in countrymirrors:
       if mir[0] == 'http' and int(mir[3]) > mingood:
         goodmirror = mir
         break
     if goodmirror:
       break

   # Check if the requested Preferred mirror is in the list
   # Note the user-requested mirror doesn't have a trailing-slash
   prefmir = None
   if preferred:
     for mir in mirrors:
       if mir[2] == preferred:
         prefmir = mir
         break
   # Otherwise pick a preferred mirror from our country
   if not prefmir and goodmirror:
     prefmir = goodmirror
   if not prefmir:  # In the worst case, choose a backup
     prefmir = random.choice(backupmirrors)

   # Record the preferred mirror and, if available, its logo and link
   # Keep the trailing-slash on the URL (it is later joined to the path_info)
   output['preferred'] = prefmir[2]
   if len(prefmir) > 5:
     output['logo'] = prefmir[4]
     output['link'] = prefmir[5]

   # Now assemble a list of all the other mirrors.
   # Keep the trailing-slash on the URL (it is later joined to the path_info)
   output['http'] = [mir[2] for mir in countrymirrors if mir[0] == 'http']
   output['ftp'] = [mir[2] for mir in countrymirrors if mir[0] == 'ftp']
   output['backup'] = [mir[2] for mir in backupmirrors]

   return output


 def mirrorwrap(environ, start_response):
   try:
     return mirrorsapp(environ, start_response)
   except:
     status = "500 Oops"
     response_headers = [("content-type","text/plain")]
     start_response(status, response_headers, sys.exc_info())
     return ["Problem running mirror.cgi, contact <infrastructure@apache.org> "
             "if it persists.\n\n"
             + traceback.format_exc() ]


 def locate_template(environ):
   # Determine the correct template by noting our filesystem location
   if environ.has_key('ASF_MIRROR_FILENAME'):
     template_file = environ['ASF_MIRROR_FILENAME'].replace(".cgi", ".html")
   elif environ.has_key('SCRIPT_FILENAME'):
     template_file = environ['SCRIPT_FILENAME'].replace(".cgi", ".html")
   else:
     template_file = sys.argv[0].replace(".cgi", ".html")

   if not os.path.isfile(template_file):
     # look in docroot instead if this is in a cgi-bin dir
     template_file = template_file.replace("/cgi-bin/", "/content/")
     if not os.path.isfile(template_file):
       template_file = DEFAULT_TEMPLATE

   return template_file


 def locate_mirrors(environ):
   # Allow the MIRRORS_LIST environment variable to override the default
   mirrors = environ.get('MIRRORS_LIST')
   if mirrors and os.path.isfile(mirrors):
     return mirrors
   return MIRRORS_LIST  # the default


 def mirrorsapp(environ, start_response):
   headers = [ ]
   resp_code = '200 OK'

   # Where is the client coming from
   region = get_region(environ)

   # Was there a preferred mirror or update requirement?
   form = cgi.FieldStorage(fp=environ['wsgi.input'],
                           environ=environ,
                           keep_blank_values=True)
   preferred = form.getfirst("Preferred", "")
   update = form.getfirst("update", "")

   # Get the last update time of the mirror database
   mirrors = locate_mirrors(environ)
   base_time = os.path.getmtime(mirrors)

   # convert from YYYYMMDDhhmm to time-since-unix-epoch
   try:
     mingood = time.mktime(time.strptime(update, "%Y%m%d%H%M"))
     # Never use a mirror more than a week old
     mingood = max(mingood, base_time - 7*24*60*60)
   except:
     # if we didn't get a time, or we can't convert it, then
     # use the time the mirror database was last updated minus 24 hours
     mingood = base_time - 24*60*60

   # Load the mirrors file and parse it out
   data = parse_mirrors(mirrors, region, preferred, mingood)

   # ======== new download tracking code ==========
   action = form.getfirst("action", "")
   filename = form.getfirst("filename", "")
   if action == 'download' and filename != '':
     url = "%s%s" % (data['preferred'][:-1], filename)
     headers.append(('Location', url))
     start_response(resp_code, headers)
     #log_download(url, data['preferred'], environ, region)
     return ''
   # ====== end new download tracking code ========

   # Note location to self
   data['location'] = environ.get('SCRIPT_NAME', DEFAULT_LOCATION)

   path_param = form.getfirst("path", None)
   if path_param:
     path_info = cgi.escape(path_param, 1)
   else:
     # Note any PATH_INFO
     if environ.has_key('PATH_INFO'):
       path_info = cgi.escape(environ['PATH_INFO'], 1)
       if environ.has_key('SCRIPT_NAME'):
         if environ['PATH_INFO'] == environ['SCRIPT_NAME']:
           path_info = ''
     else:
       path_info = ''
   # The mirror URL already has a trailing slash. Avoid doubling it up.
   if path_info.startswith('/'):
     path_info = path_info[1:]
   data['path_info'] = path_info

   template_file = locate_template(environ)

   # Print out the CGI header component
   # using xml if the filename ends with the magic '--xml' string
   if template_file.endswith('--xml.html'):
     headers.append(('Content-type', 'text/xml'))
   else:
     headers.append(('Content-type', 'text/html'))

   start_response(resp_code, headers)

   output = cStringIO.StringIO()
   template = ezt.Template(template_file)
   template.generate(output, data)
   return [ output.getvalue() ]

 # ======== new download tracking code ==========
 def log_download(url, preferred, environ, region):
   timestamp = int(time.time())
   # format: timestamp,url,mirror,IP,region,"useragent"
   try:
     fh = open('/home/jfthomps/debug', 'a')
     fh.write('%s,%s,%s,%s,%s,"%s"\n' % (timestamp, url, preferred, environ['REMOTE_ADDR'], region, environ['HTTP_USER_AGENT']))
     fh.close
   except:
     pass
 # ====== end new download tracking code ========

 if __name__ == '__main__':
   #from flup.server.fcgi import WSGIServer
   from flup.server.cgi import WSGIServer
   WSGIServer(mirrorwrap).run()
	#!/usr/local/bin/python2.7
	# -- python --
	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#
	#
	# This is a Python CGI script that uses EZT to produce templated
	# mirror content and GeoIP to choose the appropriate mirror
	#

	import sys
	import os
	import cgi
	import stat
	import random
	import time
	import traceback
	import cStringIO

	# Should be installed in the main system library
	import GeoIP

	# Insert this directory into PATH so that we can import ezt
	this_dir = os.path.dirname(__file__) or '.'
	sys.path.insert(0, this_dir)

	import ezt


	# Configurable stuff
	MIRRORS_LIST = "/x1/www/www.apache.org/mirrors/mirrors.list"
	DEFAULT_REGION = "us"
	DEFAULT_TEMPLATE = "/x1/www/www.apache.org/dyn/closer.html"
	DEFAULT_LOCATION = "http://www.apache.org/dyn/closer.cgi"


	def get_region(environ):
	"""Use GeoIP to find the client's country, falling back to
	DEFAULT_REGION on failure."""
	try:
	remote_ip = environ['REMOTE_ADDR'];
	gi = GeoIP.new(GeoIP.GEOIP_STANDARD)
	region = gi.country_code_by_addr(remote_ip).lower().strip()
	if region == 'gb':
	return 'uk'
	else:
	return region
	except:
	### should we log an error here? absorbing without reporting is
	### generally bad form.
	return DEFAULT_REGION


	def parse_mirrors(filename, country, preferred, mingood):
	"""Parse the mirror database to find the best mirrors for a client.

	The Format of the mirror database is (last two fields are optional):
	ftp au ftp://ftp.planetmirror.com/pub/apache/dist/ 1117724635 http://example.com/logo.gif http://example.com/ """

	output = { 'http' : [ ],
	'ftp' : [ ],
	'backup' : [ ],
	'preferred' : None,
	'logo' : None,
	'link' : None,
	}

	# Read the mirror database and put it in a list of lists
	# skip empty lines and comment
	mirrors = [line.split() for line in open(filename).readlines()
	if line.strip() and not line.startswith('#')]

	mirrors.append(['http', 'us', 'http://archive.apache.org/dist/', '9999999999'])

	# Add trailing slashes where missing. Otherwise,
	# strcat("http://www.mirror.org", pathinfo=".foo.evil") would link to
	# http://www.mirror.org.foo.evil
	for mir in mirrors:
	if not mir[2].endswith('/'):
	mir[2] += '/'

	# grab the backup mirrors
	backupmirrors = [mir for mir in mirrors if mir[1] == 'Backup']

	# Grab the mirrors for the requested country or, failing that,
	# from the default region (us)
	for region in (country, DEFAULT_REGION):
	countrymirrors = [mir for mir in mirrors if mir[1] == region]
	random.shuffle(countrymirrors)
	goodmirror = None
	for mir in countrymirrors:
	if mir[0] == 'http' and int(mir[3]) > mingood:
	goodmirror = mir
	break
	if goodmirror:
	break

	# Check if the requested Preferred mirror is in the list
	# Note the user-requested mirror doesn't have a trailing-slash
	prefmir = None
	if preferred:
	for mir in mirrors:
	if mir[2] == preferred:
	prefmir = mir
	break
	# Otherwise pick a preferred mirror from our country
	if not prefmir and goodmirror:
	prefmir = goodmirror
	if not prefmir: # In the worst case, choose a backup
	prefmir = random.choice(backupmirrors)

	# Record the preferred mirror and, if available, its logo and link
	# Keep the trailing-slash on the URL (it is later joined to the path_info)
	output['preferred'] = prefmir[2]
	if len(prefmir) > 5:
	output['logo'] = prefmir[4]
	output['link'] = prefmir[5]

	# Now assemble a list of all the other mirrors.
	# Keep the trailing-slash on the URL (it is later joined to the path_info)
	output['http'] = [mir[2] for mir in countrymirrors if mir[0] == 'http']
	output['ftp'] = [mir[2] for mir in countrymirrors if mir[0] == 'ftp']
	output['backup'] = [mir[2] for mir in backupmirrors]

	return output


	def mirrorwrap(environ, start_response):
	try:
	return mirrorsapp(environ, start_response)
	except:
	status = "500 Oops"
	response_headers = [("content-type","text/plain")]
	start_response(status, response_headers, sys.exc_info())
	return ["Problem running mirror.cgi, contact <infrastructure@apache.org> "
	"if it persists.\n\n"
	+ traceback.format_exc() ]


	def locate_template(environ):
	# Determine the correct template by noting our filesystem location
	if environ.has_key('ASF_MIRROR_FILENAME'):
	template_file = environ['ASF_MIRROR_FILENAME'].replace(".cgi", ".html")
	elif environ.has_key('SCRIPT_FILENAME'):
	template_file = environ['SCRIPT_FILENAME'].replace(".cgi", ".html")
	else:
	template_file = sys.argv[0].replace(".cgi", ".html")

	if not os.path.isfile(template_file):
	# look in docroot instead if this is in a cgi-bin dir
	template_file = template_file.replace("/cgi-bin/", "/content/")
	if not os.path.isfile(template_file):
	template_file = DEFAULT_TEMPLATE

	return template_file


	def locate_mirrors(environ):
	# Allow the MIRRORS_LIST environment variable to override the default
	mirrors = environ.get('MIRRORS_LIST')
	if mirrors and os.path.isfile(mirrors):
	return mirrors
	return MIRRORS_LIST # the default


	def mirrorsapp(environ, start_response):
	headers = [ ]
	resp_code = '200 OK'

	# Where is the client coming from
	region = get_region(environ)

	# Was there a preferred mirror or update requirement?
	form = cgi.FieldStorage(fp=environ['wsgi.input'],
	environ=environ,
	keep_blank_values=True)
	preferred = form.getfirst("Preferred", "")
	update = form.getfirst("update", "")

	# Get the last update time of the mirror database
	mirrors = locate_mirrors(environ)
	base_time = os.path.getmtime(mirrors)

	# convert from YYYYMMDDhhmm to time-since-unix-epoch
	try:
	mingood = time.mktime(time.strptime(update, "%Y%m%d%H%M"))
	# Never use a mirror more than a week old
	mingood = max(mingood, base_time - 72460*60)
	except:
	# if we didn't get a time, or we can't convert it, then
	# use the time the mirror database was last updated minus 24 hours
	mingood = base_time - 246060

	# Load the mirrors file and parse it out
	data = parse_mirrors(mirrors, region, preferred, mingood)

	# ======== new download tracking code ==========
	action = form.getfirst("action", "")
	filename = form.getfirst("filename", "")
	if action == 'download' and filename != '':
	url = "%s%s" % (data['preferred'][:-1], filename)
	headers.append(('Location', url))
	start_response(resp_code, headers)
	#log_download(url, data['preferred'], environ, region)
	return ''
	# ====== end new download tracking code ========

	# Note location to self
	data['location'] = environ.get('SCRIPT_NAME', DEFAULT_LOCATION)

	path_param = form.getfirst("path", None)
	if path_param:
	path_info = cgi.escape(path_param, 1)
	else:
	# Note any PATH_INFO
	if environ.has_key('PATH_INFO'):
	path_info = cgi.escape(environ['PATH_INFO'], 1)
	if environ.has_key('SCRIPT_NAME'):
	if environ['PATH_INFO'] == environ['SCRIPT_NAME']:
	path_info = ''
	else:
	path_info = ''
	# The mirror URL already has a trailing slash. Avoid doubling it up.
	if path_info.startswith('/'):
	path_info = path_info[1:]
	data['path_info'] = path_info

	template_file = locate_template(environ)

	# Print out the CGI header component
	# using xml if the filename ends with the magic '--xml' string
	if template_file.endswith('--xml.html'):
	headers.append(('Content-type', 'text/xml'))
	else:
	headers.append(('Content-type', 'text/html'))

	start_response(resp_code, headers)

	output = cStringIO.StringIO()
	template = ezt.Template(template_file)
	template.generate(output, data)
	return [ output.getvalue() ]

	# ======== new download tracking code ==========
	def log_download(url, preferred, environ, region):
	timestamp = int(time.time())
	# format: timestamp,url,mirror,IP,region,"useragent"
	try:
	fh = open('/home/jfthomps/debug', 'a')
	fh.write('%s,%s,%s,%s,%s,"%s"\n' % (timestamp, url, preferred, environ['REMOTE_ADDR'], region, environ['HTTP_USER_AGENT']))
	fh.close
	except:
	pass
	# ====== end new download tracking code ========

	if __name__ == '__main__':
	#from flup.server.fcgi import WSGIServer
	from flup.server.cgi import WSGIServer
	WSGIServer(mirrorwrap).run()