#!/usr/bin/env python
# -*- coding: utf-8 -*-

################## ChangeLog ##################
## v0.1 Prototype		             ##
## v0.2 Added version search for Typo3 	     ##
## v0.3 Added version guessing		     ##
## v0.4 Optimized requests 		     ##
## v0.5 Added support for Typo v6.X  	     ##
## v0.6 Added extension search		     ##
## v0.7 Added version search for extensions  ##
## v0.8 Added support for TOR Service	     ##
###############################################

############ Version information ##############
__version__ = "0.8.1"
__program__ = "Typo-Enumerator v" + __version__
__description__ = 'Find out the Typo3 Version, Login-URL and Extensions'
__author__ = "Jan Rude"
__licence__ = "BSD Licence"
__status__ = "Development"  # ("Prototype", "Development", "Final")
###############################################

################## Imports ####################
import os
import re
import gzip
import time
import socket
import urllib
import urllib2
import requests
import argparse
import datetime
from Queue import Queue
from colorama import Fore
from os.path import isfile
from operator import itemgetter
from threading import Thread, Lock
from collections import OrderedDict
import xml.etree.ElementTree as ElementTree
###############################################

############### Global variables ##############
user_agent = {'User-Agent' : None}
extension_list = []
verbosity = False
###############################################

# Searching for Typo3 version in ChangeLog
def check_typo_version_ChangeLog(domain):
	global user_agent
	try:
		url = urllib2.Request('http://' + domain + '/typo3_src/ChangeLog', None, user_agent)
		f = urllib2.urlopen(url, timeout = 3.0)
		changelog = f.read(200)
		f.close()
		regex = re.compile("RELEASE] Release of (.*)")
		searchVersion = regex.search(changelog)
		version = searchVersion.groups()
		print "Typo3 Version:".ljust(32) + Fore.GREEN + version[0] + Fore.RESET
		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[1]
	except Exception, e:
		check_typo_version_NEWS_TXT(domain)

# Searching for Typo3 version in NEWS.txt
def check_typo_version_NEWS_TXT(domain):
	global user_agent
	try:
		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.txt', None, user_agent)
		f = urllib2.urlopen(url, timeout = 3.0)
		changelog = f.read(500)
		f.close()
		regex = re.compile("This document contains information about (.*) which")
		searchVersion = regex.search(changelog)
		version = searchVersion.groups()
		print "Typo3 Version:".ljust(32), Fore.GREEN + version[0] + '.XX' + Fore.RED + ' (only guessable)'+ Fore.RESET
		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[2]
	except:
		check_typo_version_NEWS_MD(domain)

# Searching for Typo3 version in NEWS.md
def check_typo_version_NEWS_MD(domain):
	global user_agent
	try:
		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.md', None, user_agent)
		f = urllib2.urlopen(url, timeout = 3.0)
		changelog = f.read(80)
		f.close()
		regex = re.compile("(.*) - WHAT'S NEW")
		searchVersion = regex.search(changelog)
		version = searchVersion.groups()
		print "Typo3 Version:\t\t", Fore.GREEN + version[0] + '.XX' + Fore.RED + ' (only guessable)'+ Fore.RESET
		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[2]
	except:
		print "Typo3 Version:".ljust(32) + Fore.RED + "Not found" + Fore.RESET

# Searching Typo3 login page
def check_typo_login(domain):
	global user_agent
	try:
		return check_main_page(domain)
		r = requests.get('http://' + domain + '/typo3/index.php', allow_redirects=False, timeout=8.0, headers=user_agent)
		statusCode = r.status_code
		httpResponse = r.text
		if statusCode == 200:
			return check_title(httpResponse, r.url)
		elif (statusCode == 301) or (statusCode == 302):
			location = r.headers['location']
			if ("http://") in location:
				new = location.split("//")
				new2 = new[1].split("/")
				check_typo_login(new2[0])
			elif ("https://") in location:
				r = requests.get(location, timeout=8.0, headers=user_agent, verify=False)
				statusCode = r.status_code
				httpResponse = r.text
				return check_title(httpResponse, r.url)
		elif statusCode == 404:
			return check_main_page(domain)
		else:
			print "Oops! Got:".ljust(32) + str(statusCode) + ": " + str(r.raise_for_status())
	except requests.exceptions.Timeout:
		print Fore.RED + "Connection timed out" + Fore.RESET
	except requests.exceptions.TooManyRedirects:
		print Fore.RED + "Too many redirects" + Fore.RESET
	except requests.exceptions.RequestException as e:
		print Fore.RED + str(e) + Fore.RESET

# Searching for Typo3 references in title
def check_title(response, url):
	regex = re.compile("<title>(.*)</title>", re.IGNORECASE)
	searchTitle = regex.search(response)
	title = searchTitle.groups()[0]
	if 'TYPO3' in title or 'TYPO3 SVN ID:' in response:
		print "Typo3 Login:".ljust(32) + Fore.GREEN + url + Fore.RESET
		return True
	else:
		print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
		return False

# Searching for Typo3 references in HTML comments
def check_main_page(domain):
	req = urllib2.Request('http://' + domain, None, user_agent)
	connection = urllib2.urlopen(req)
	response = connection.read()
	connection.close()
	if 'fe_typo_user' in connection.info().getheader('Set-Cookie'):
		print "Typo3 Login:".ljust(32) + Fore.GREEN + "Typo3 is used, but could not find login" + Fore.RESET
		return True
	else:
		try:
			regex = re.compile("This website is powered by TYPO3(.*)", re.IGNORECASE)
			searchHTML = regex.search(response)
			searchHTML.groups()[0]
			print "Typo3 Login:".ljust(32) + Fore.GREEN + "Typo3 is used, but could not find login" + Fore.RESET
			return True
		except:
			print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
	return False

# Searching installed extensions
def check_extensions(domain, input_queue, output_queue):
	global user_agent
	global verbosity
	while True:
		extension = input_queue.get()
		try:
			req = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + "/", None, user_agent)
			connection = urllib2.urlopen(req)
			connection.close()
			print "TEST:"
			check_extension_version(domain, extension, output_queue)
		except urllib2.HTTPError, e:
			print "CODE:", e.code
			if e.code == 403:
				check_extension_version(domain, extension, output_queue)
			elif e.code == 404:
				if verbosity:
					output_queue.put(extension.ljust(32) + Fore.RED + "not installed" + Fore.RESET)
				pass
		except urllib2.URLError, e:
			print "CODE:", e
			print str(e.reason)
		except Exception, e:
				import traceback
				print ('generic exception: ', traceback.format_exc())
		input_queue.task_done()

# Searching version of installed extension
def check_extension_version(domain, extension, output_queue):
	global verbosity
	global user_agent
	try:
		url = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + '/ChangeLog', None, user_agent)
		connection = urllib2.urlopen(url, timeout = 15.0)
		changelog = connection.read(1500)
		connection.close()
		regex = re.compile("(\d{1,2}\.\d{1,2}\.[0-9][0-9]?[' '\n])")
		searchVersion = regex.search(changelog)
		version = searchVersion.groups()
		output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (v" + version[0].split()[0] + ")" + Fore.RESET)
	except:
		try:
			regex = re.compile("(\d{2,4}[\.\-]\d{1,2}[\.\-]\d{1,4})")
			searchVersion = regex.search(changelog)
			version = searchVersion.groups()
			output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (last entry from " + version[0] + ")" + Fore.RESET)
		except:
			if verbosity:
				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (no version information found)")
			else:
				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)

# Output thread
def output_thread(q):
	if q.empty():
		print Fore.RED + "No extensions are installed" + Fore.RESET
	else:	
		while q is not q.empty():
			try:
				extension = q.get()
				print(extension)
				q.task_done()
			except Exception, e:
				print "Oops! Got:", e

# Loading extensions
def generate_extensions_list(top):
	global extension_list
	extension = 'extensions.xml'
	print "\nLoading extensions..."
	if not isfile(extension):
	 	print(Fore.RED + "File not found: " + extension + "\nAborting..." + Fore.RESET)
	 	sys.exit(-2)
	tree = ElementTree.parse(extension) 
	tag_dict = tree.getroot()
	exten_Dict = {}
	for extensions in tag_dict.getchildren():
			ext = {extensions.get('extensionkey'):extensions[0].text}
			exten_Dict.update(ext)
	print 'Loaded ' , len(exten_Dict), ' extensions\n'
	if top is not None:
		sorted_dict = sorted(exten_Dict.iteritems(), key=lambda x: int(x[1]), reverse=True)		
		for i in xrange(0,top):
			extension_list.append(sorted_dict[i][0])
	else:
		for extension_name in tag_dict:
			extension_list.append(extension_name.get('extensionkey'))

# Copy selected extensions in queue
def copy_extensions(input_queue):
	global extension_list
	for ext in extension_list:
		input_queue.put(ext)

# Progressbar
def dlProgress(count, blockSize, totalSize):
	percent = int(count*blockSize*100/totalSize)
	sys.stdout.write("\rDownloading extentions: " + "%d%%" % percent)
	sys.stdout.flush()

# Update function
def update():
	try:
		urllib.urlretrieve('http://ter.sitedesign.dk/ter/extensions.xml.gz', 'extensions.gz', reporthook=dlProgress)
		inf = gzip.open('extensions.gz', 'rb')
		file_content = inf.read()
		inf.close()
		outF = file("extensions.xml", 'wb')
		outF.write(file_content)
		outF.close()
		print "\n"
		os.remove('extensions.gz')
	except Exception, e:
		print "Oops! Got:".ljust(32), e

# Using Privoxy and TOR for all connections
def setting_up_tor():
	try:
		import socks
	except:
		print "The module 'SocksiPy' is not installed.\nPlease install it with: sudo apt-get install python-socksipy"
		sys.exit(-2)
	print "Checking connection to TOR through Privoxy"
	socks.setdefaultproxy(socks.PROXY_TYPE_HTTP, "127.0.0.1", 8118, True)
	socket.socket = socks.socksocket
	try:
		url = urllib2.Request('https://check.torproject.org/')
		torcheck = urllib2.urlopen(url)
		response = torcheck.read()
		torcheck.close()
	except:
		print "Failed to connect to Privoxy and/or TOR!\nPlease make sure they are running and configured!\nYou can start them with:\nservice privoxy start\nservice tor start\n"
		sys.exit(-2)
	try:
		regex = re.compile('Congratulations. This browser is configured to use Tor.')
		searchVersion = regex.search(response)
		version = searchVersion.groups()
		print "Connection to TOR established"
	except:
		print "It seems like TOR is not used.\nAborting...\n"
		sys.exit(-2)

# Startmethod
def start(domain, top, threads):
	in_queue = Queue()
	out_queue = Queue()
	regex = re.compile("(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
	searchIP = regex.search(domain)
	if not (searchIP is None):
		IP = searchIP.groups()[0]
		hostname = socket.gethostbyaddr(IP)
		print("\n\n[*] Check for " + domain + " (" + hostname[0] + ")")
	else:
		print("\n\n[*] Check for " + domain)		
	if check_typo_login(domain) is True:
		if not extension_list:
			generate_extensions_list(top)
		check_typo_version_ChangeLog(domain)
		copy_extensions(in_queue)
		print '\nChecking', in_queue.qsize(), 'Extensions:\nThis may take a while...'
		for i in xrange(0, threads):
			t = Thread(target=check_extensions, args=(domain, in_queue, out_queue))
			t.daemon = True
			t.start()
		in_queue.join()
		t = Thread(target=output_thread, args=(out_queue,))
		t.daemon = True
		t.start()
		out_queue.join()

# Main
def main(argv):
	global user_agent
	global verbosity
	parser = argparse.ArgumentParser(add_help=False, usage='typoenum.py -d DOMAIN [DOMAIN ...] | -f FILE [--user_agent USER-AGENT] [--top VALUE] [-v] [--tor]')
	group = parser.add_mutually_exclusive_group()
	group.add_argument('-d', '--domain', dest='domain', type=str, nargs='+')
	group.add_argument('-f', '--file', dest='file', help='File with a list of domains')
	group.add_argument('-u', '--update', dest='update', action='store_true',help='Get/Update the extension file')
	parser.add_argument('--user_agent', dest='user_agent', default='Mozilla/5.0', metavar='USER-AGENT (default: Mozilla/5.0)')
	parser.add_argument('--top', type=int, dest='top', metavar='VALUE', help='Check only most X downloaded extensions (default: all)', default=None)
	parser.add_argument('-v', '--verbose', help='increase output verbosity', action='store_true')
	parser.add_argument('--tor', help='using tor for connections', action='store_true')
	parser.add_argument('-t', '--threads', dest='threads', default=10, type=int, help='(default: 10)')
	args = parser.parse_args()

	if not args.domain and not args.file and not args.update:
		parser.print_help()
		return True

	if args.tor:
		setting_up_tor()
		
	if args.update:
		update()
		return True

	user_agent = {'User-Agent' : args.user_agent}
	verbosity = args.verbose

	if args.domain and not args.file:
		for dom in args.domain:
			start(dom, args.top, args.threads)

	elif not args.domain and args.file:
		if not isfile(args.file):
			print(Fore.RED + "\nFile not found: " + args.file + "\nAborting..." + Fore.RESET)
			sys.exit(-2)
		else:
			with open(args.file, 'r') as f:
				for line in f:
					start(line.strip(), args.top, args.threads)
	print '\n'
	now = datetime.datetime.now()
	print __program__ + ' finished at ' + now.strftime("%Y-%m-%d %H:%M:%S") + '\n'
	return True

if __name__ == "__main__":
	import sys
	print('\n' + 70*'*')
	print('\t' + __program__ )
	print('\t' + __description__)
	print('\t' + '(c)2014 by ' + __author__)
	print('\t' + 'Status:\t' + __status__)
	print('\t' + 'For legal purposes only!')
	print(70*'*' + '\n')
	sys.exit( not main( sys.argv ) )