Update typoenum.py

typoenum.py

@@ -1,369 +1,786 @@
 #!/usr/bin/env python
+
 # -*- coding: utf-8 -*-
 
+
+
 ################## ChangeLog ##################
+
 ## v0.1 Prototype			     ##
+
 ## v0.2 Added version search for Typo3 	     ##
+
 ## v0.3 Added version guessing		     ##
+
 ## v0.4 Optimized requests 		     ##
+
 ## v0.5 Added support for Typo v6.X  	     ##
+
 ## v0.6 Added extension search		     ##
+
 ## v0.7 Added version search for extensions  ##
-## v0.8 Added support for TOR Service 	     ##
+
+## v0.8 Added support for TOR Service 		 ##
+
 ###############################################
 
+
+
 ############ Version information ##############
-__version__ = "0.8"
+
+__version__ = "0.8.1"
+
 __program__ = "Typo-Enumerator v" + __version__
+
 __description__ = 'Find out the Typo3 Version, Login-URL and Extensions'
+
 __author__ = "Jan Rude"
+
 __licence__ = "BSD Licence"
+
 __status__ = "Development"  # ("Prototype", "Development", "Final")
+
 ###############################################
 
+
+
 ################## Imports ####################
+
 import os
+
 import re
+
 import gzip
+
 import time
+
 import socket
+
 import urllib
+
 import urllib2
+
+import requests
+
 import argparse
+
 import datetime
+
 from Queue import Queue
+
 from colorama import Fore
+
 from os.path import isfile
+
 from operator import itemgetter
+
 from threading import Thread, Lock
+
 from collections import OrderedDict
+
 import xml.etree.ElementTree as ElementTree
+
 ###############################################
 
+
+
 ############### Global variables ##############
+
 user_agent = {'User-Agent' : None}
+
 extension_list = []
+
 verbosity = False
+
 ###############################################
 
+
+
 # Checks the Typo version
+
 def check_typo_version_ChangeLog(domain):
-	global verbosity
+
 	global user_agent
+
 	try:
+
 		url = urllib2.Request('http://' + domain + '/typo3_src/ChangeLog', None, user_agent)
+
 		f = urllib2.urlopen(url, timeout = 3.0)
+
 		changelog = f.read(200)
+
 		f.close()
+
 		regex = re.compile("RELEASE] Release of (.*)")
+
 		searchVersion = regex.search(changelog)
+
 		version = searchVersion.groups()
+
 		print "Typo3 Version:".ljust(32) + Fore.GREEN + version[0] + Fore.RESET
+
 		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[1]
+
 	except Exception, e:
-		if verbosity:
+
-			print "Typo3 Version:".ljust(32) + "first check failed, trying second one..."
 		check_typo_version_NEWS_TXT(domain)
 
+
+
 def check_typo_version_NEWS_TXT(domain):
-	global verbosity
+
 	global user_agent
+
 	try:
+
 		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.txt', None, user_agent)
+
 		f = urllib2.urlopen(url, timeout = 3.0)
+
 		changelog = f.read(500)
+
 		f.close()
+
 		regex = re.compile("This document contains information about (.*) which")
+
 		searchVersion = regex.search(changelog)
+
 		version = searchVersion.groups()
+
 		print "Typo3 Version:".ljust(32), Fore.GREEN + version[0] + '.XX' + Fore.RED + ' (only guessable)'+ Fore.RESET
+
 		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[2]
+
 	except:
-		if verbosity:
+
-			print "Typo3 Version:".ljust(32) + "second check failed, trying third one..."
 		check_typo_version_NEWS_MD(domain)
 
+
+
 def check_typo_version_NEWS_MD(domain):
+
 	global user_agent
+
 	try:
+
 		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.md', None, user_agent)
+
 		f = urllib2.urlopen(url, timeout = 3.0)
+
 		changelog = f.read(80)
+
 		f.close()
+
 		regex = re.compile("(.*) - WHAT'S NEW")
+
 		searchVersion = regex.search(changelog)
+
 		version = searchVersion.groups()
+
 		print "Typo3 Version:\t\t", Fore.GREEN + version[0] + '.XX' + Fore.RED + ' (only guessable)'+ Fore.RESET
+
 		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + version[0].split()[2]
+
 	except:
+
 		print "Typo3 Version:".ljust(32) + Fore.RED + "Not found" + Fore.RESET
 
+
+
 # Checks the Typo login
+
 def check_typo_login(domain):
+
 	global user_agent
+
 	try:
-		req = urllib2.Request('http://' + domain + '/typo3/index.php', None, user_agent)
+
-		connection = urllib2.urlopen(req)
+		return check_main_page(domain)
-		response = connection.read()
+
-		return check_title(response, connection.geturl())
+
-		connection.close()
+
-	except urllib2.HTTPError, e:
+		r = requests.get('http://' + domain + '/typo3/index.php', allow_redirects=False, timeout=8.0, headers=user_agent)
-		if e.code == 403:
+
-			return check_title(response, connection.geturl())
+		statusCode = r.status_code
-		elif e.code == 404:
+
-			print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
+		httpResponse = r.text
-	except urllib2.URLError, e:
+
-		print str(e.reason)
+		if statusCode == 200:
-	except Exception, e:
+
-		import traceback
+			return check_title(httpResponse, r.url)
-		print ('generic exception: ', traceback.format_exc())
+
+		elif (statusCode == 301) or (statusCode == 302):
+
+			location = r.headers['location']
+
+			if ("http://") in location:
+
+				new = location.split("//")
+
+				new2 = new[1].split("/")
+
+				check_typo_login(new2[0])
+
+			elif ("https://") in location:
+
+				r = requests.get(location, timeout=8.0, headers=user_agent, verify=False)
+
+				statusCode = r.status_code
+
+				httpResponse = r.text
+
+				return check_title(httpResponse, r.url)
+
+		elif statusCode == 404:
+
+			return check_main_page(domain)
+
+		else:
+
+			print "Oops! Got:".ljust(32) + str(statusCode) + ": " + str(r.raise_for_status())
+
+	except requests.exceptions.Timeout:
+
+		print Fore.RED + "Connection timed out" + Fore.RESET
+
+	except requests.exceptions.TooManyRedirects:
+
+		print Fore.RED + "Too many redirects" + Fore.RESET
+
+	except requests.exceptions.RequestException as e:
+
+		print Fore.RED + str(e) + Fore.RESET
+
 
 
 # Checks, if URL is a Typo-Login
-def check_title(httpResponse, url):
+
+def check_title(response, url):
+
 	regex = re.compile("<title>(.*)</title>", re.IGNORECASE)
-	searchTitle = regex.search(httpResponse)
+
+	searchTitle = regex.search(response)
+
 	title = searchTitle.groups()[0]
-	if 'TYPO3' in title or 'TYPO3 SVN ID:' in httpResponse:
+
+	if 'TYPO3' in title or 'TYPO3 SVN ID:' in response:
+
 		print "Typo3 Login:".ljust(32) + Fore.GREEN + url + Fore.RESET
+
 		return True
+
 	else:
+
 		print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
+
 		return False
 
-# Searches for installed extensions
+
-def check_extensions(domain, input_queue, output_queue):
+
-	global user_agent
+def check_main_page(domain):
-	global verbosity
+
-	while True:
+	req = urllib2.Request('http://' + domain, None, user_agent)
-		extension = input_queue.get()
+
+	connection = urllib2.urlopen(req)
+
+	response = connection.read()
+
+	connection.close()
+
+	if 'fe_typo_user' in connection.info().getheader('Set-Cookie'):
+
+		print "Typo3 Login:".ljust(32) + Fore.GREEN + "Typo3 is used, but could not find login" + Fore.RESET
+
+		return True
+
+	else:
+
 		try:
+
+			regex = re.compile("This website is powered by TYPO3(.*)", re.IGNORECASE)
+
+			searchHTML = regex.search(response)
+
+			searchHTML.groups()[0]
+
+			print "Typo3 Login:".ljust(32) + Fore.GREEN + "Typo3 is used, but could not find login" + Fore.RESET
+
+			return True
+
+		except:
+
+			print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
+
+	return False
+
+
+
+# Searches for installed extensions
+
+def check_extensions(domain, input_queue, output_queue):
+
+	global user_agent
+
+	global verbosity
+
+	while True:
+
+		extension = input_queue.get()
+
+		try:
+
 			req = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + "/", None, user_agent)
+
 			connection = urllib2.urlopen(req)
+
 			connection.close()
+
+			print "TEST:"
+
 			check_extension_version(domain, extension, output_queue)
+
 		except urllib2.HTTPError, e:
+
+			print "CODE:", e.code
+
 			if e.code == 403:
+
 				check_extension_version(domain, extension, output_queue)
+
 			elif e.code == 404:
+
 				if verbosity:
+
 					output_queue.put(extension.ljust(32) + Fore.RED + "not installed" + Fore.RESET)
+
 				pass
+
 		except urllib2.URLError, e:
+
+			print "CODE:", e
+
 			print str(e.reason)
+
 		except Exception, e:
+
 				import traceback
+
 				print ('generic exception: ', traceback.format_exc())
+
 		input_queue.task_done()
 
+
+
 # Searches for version of installed extension
+
 def check_extension_version(domain, extension, output_queue):
+
 	global verbosity
+
 	global user_agent
+
 	try:
+
 		url = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + '/ChangeLog', None, user_agent)
+
 		connection = urllib2.urlopen(url, timeout = 15.0)
+
 		changelog = connection.read(1500)
+
 		connection.close()
+
 		regex = re.compile("(\d{1,2}\.\d{1,2}\.[0-9][0-9]?[' '\n])")
+
 		searchVersion = regex.search(changelog)
+
 		version = searchVersion.groups()
+
 		output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (v" + version[0].split()[0] + ")" + Fore.RESET)
+
 	except:
+
 		try:
+
 			regex = re.compile("(\d{2,4}[\.\-]\d{1,2}[\.\-]\d{1,4})")
+
 			searchVersion = regex.search(changelog)
+
 			version = searchVersion.groups()
+
 			output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (last entry from " + version[0] + ")" + Fore.RESET)
+
 		except:
+
 			if verbosity:
-				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (could not find version)")
+
+				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (no version information found)")
+
 			else:
+
 				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)
 
+
+
 # Output
+
 def output_thread(q):
+
 	if q.empty():
+
 		print Fore.RED + "No extensions are installed" + Fore.RESET
+
 	else:	
+
 		while q is not q.empty():
+
 			try:
+
 				extension = q.get()
+
 				print(extension)
+
 				q.task_done()
+
 			except Exception, e:
+
 				print "Oops! Got:", e
 
+
+
 # Loading extensions
+
 def generate_extensions_list(top):
+
 	global extension_list
+
 	extension = 'extensions.xml'
+
 	print "\nLoading extensions..."
+
 	if not isfile(extension):
+
 	 	print(Fore.RED + "File not found: " + extension + "\nAborting..." + Fore.RESET)
+
 	 	sys.exit(-2)
 
+
+
 	tree = ElementTree.parse(extension) 
+
 	tag_dict = tree.getroot()
+
 	exten_Dict = {}
 
+
+
 	for extensions in tag_dict.getchildren():
+
 			ext = {extensions.get('extensionkey'):extensions[0].text}
+
 			exten_Dict.update(ext)
+
 	print 'Loaded ' , len(exten_Dict), ' extensions\n'
 
 	if top is not None:
+
 		sorted_dict = sorted(exten_Dict.iteritems(), key=lambda x: int(x[1]), reverse=True)		
 
 		for i in xrange(0,top):
+
 			extension_list.append(sorted_dict[i][0])
+
 	else:
+
 		for extension_name in tag_dict:
+
 			extension_list.append(extension_name.get('extensionkey'))
 
+
+
 # Copy used extensions in queue
+
 def copy_extensions(input_queue):
+
 	global extension_list
+
 	for ext in extension_list:
+
 		input_queue.put(ext)
 
+
+
 # Progressbar
+
 def dlProgress(count, blockSize, totalSize):
+
 	percent = int(count*blockSize*100/totalSize)
+
 	sys.stdout.write("\rDownloading extentions: " + "%d%%" % percent)
+
 	sys.stdout.flush()
 
+
+
 # Update function
+
 def update():
+
 	try:
+
 		urllib.urlretrieve('http://ter.sitedesign.dk/ter/extensions.xml.gz', 'extensions.gz', reporthook=dlProgress)
+
 		inf = gzip.open('extensions.gz', 'rb')
+
 		file_content = inf.read()
+
 		inf.close()
+
 		outF = file("extensions.xml", 'wb')
+
 		outF.write(file_content)
+
 		outF.close()
+
 		print "\n"
+
 		os.remove('extensions.gz')
+
 	except Exception, e:
+
 		print "Oops! Got:".ljust(32), e
 
+
+
 # Using Privoxy and TOR for all connections
+
 def setting_up_tor():
+
 	try:
+
 		import socks
+
 	except:
+
 		print "The module 'SocksiPy' is not installed.\nPlease install it with: sudo apt-get install python-socksipy"
+
 		sys.exit(-2)
 
-	print "Setting up proxy to Privoxy on Port 8118"
+
+
+	print "Checking connection to TOR through Privoxy"
+
 	socks.setdefaultproxy(socks.PROXY_TYPE_HTTP, "127.0.0.1", 8118, True)
+
 	socket.socket = socks.socksocket
+
 	try:
+
 		url = urllib2.Request('https://check.torproject.org/')
+
 		torcheck = urllib2.urlopen(url)
+
 		response = torcheck.read()
+
 		torcheck.close()
+
 	except:
+
 		print "Failed to connect to Privoxy and/or TOR!\nPlease make sure they are running and configured!\nYou can start them with:\nservice privoxy start\nservice tor start\n"
+
 		sys.exit(-2)
+
 	try:
+
 		regex = re.compile('Congratulations. This browser is configured to use Tor.')
+
 		searchVersion = regex.search(response)
+
 		version = searchVersion.groups()
+
 		print "Connection to TOR established"
+
 	except:
+
 		print "It seems like TOR is not used.\nAborting...\n"
+
 		sys.exit(-2)
 
+
+
 # Starting checks
-def start(domain, top, tor):
+
+def start(domain, top, threads):
+
 	in_queue = Queue()
+
 	out_queue = Queue()
+
 	regex = re.compile("(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
+
 	searchIP = regex.search(domain)
+
 	if not (searchIP is None):
+
 		IP = searchIP.groups()[0]
+
 		hostname = socket.gethostbyaddr(IP)
+
 		print("\n\n[*] Check for " + domain + " (" + hostname[0] + ")")
+
 	else:
+
 		print("\n\n[*] Check for " + domain)		
 
+		
+
 	if check_typo_login(domain) is True:
+
 		if not extension_list:
+
 			generate_extensions_list(top)
 
+		
+
 		check_typo_version_ChangeLog(domain)
+
 		copy_extensions(in_queue)
+
 		print '\nChecking', in_queue.qsize(), 'Extensions:\nThis may take a while...'
-		if tor:
+
-			threads = 2
-		else:
-			threads = 10
 		for i in xrange(0, threads):
+
 			t = Thread(target=check_extensions, args=(domain, in_queue, out_queue))
+
 			t.daemon = True
+
 			t.start()
+
 		in_queue.join()
 
+
+
 		t = Thread(target=output_thread, args=(out_queue,))
+
 		t.daemon = True
+
 		t.start()
+
 		out_queue.join()
 
+
+
 # Main
+
 def main(argv):
+
 	global user_agent
+
 	global verbosity
+
 	parser = argparse.ArgumentParser(add_help=False, usage='typoenum.py -d DOMAIN [DOMAIN ...] | -f FILE [--user_agent USER-AGENT] [--top VALUE] [-v] [--tor]')
+
 	group = parser.add_mutually_exclusive_group()
+
 	group.add_argument('-d', '--domain', dest='domain', type=str, nargs='+')
+
 	group.add_argument('-f', '--file', dest='file', help='File with a list of domains')
+
 	group.add_argument('-u', '--update', dest='update', action='store_true',help='Get/Update the extension file')
+
 	parser.add_argument('--user_agent', dest='user_agent', default='Mozilla/5.0', metavar='USER-AGENT (default: Mozilla/5.0)')
+
 	parser.add_argument('--top', type=int, dest='top', metavar='VALUE', help='Check only most X downloaded extensions (default: all)', default=None)
-	parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
+
-	parser.add_argument("--tor", help="using tor for connections", action="store_true")
+	parser.add_argument('-v', '--verbose', help='increase output verbosity', action='store_true')
+
+	parser.add_argument('--tor', help='using tor for connections', action='store_true')
+
+	parser.add_argument('-t', '--threads', dest='threads', default=10, type=int, help='(default: 10)')
+
 	args = parser.parse_args()
 
+	
+
 	if not args.domain and not args.file and not args.update:
+
 		parser.print_help()
+
 		return True
 
+
+
 	if args.tor:
+
 		setting_up_tor()
 
+
+
 	if args.update:
+
 		update()
+
 		return True
 
+
+
 	user_agent = {'User-Agent' : args.user_agent}
+
 	verbosity = args.verbose
 
+
+
 	if args.domain and not args.file:
+
 		for dom in args.domain:
-			start(dom, args.top, args.tor)
+
+			start(dom, args.top, args.threads)
+
+
 
 	elif not args.domain and args.file:
+
 		if not isfile(args.file):
+
 			print(Fore.RED + "\nFile not found: " + args.file + "\nAborting..." + Fore.RESET)
+
 			sys.exit(-2)
+
 		else:
+
 			with open(args.file, 'r') as f:
+
 				for line in f:
-					start(line.strip(), args.top, args.tor)
+
+					start(line.strip(), args.top, args.threads)
+
 	print '\n'
+
 	now = datetime.datetime.now()
+
 	print __program__ + ' finished at ' + now.strftime("%Y-%m-%d %H:%M:%S") + '\n'
+
 	return True
 
+
+
 if __name__ == "__main__":
+
 	import sys
+
 	print('\n' + 70*'*')
+
 	print('\t' + __program__ )
+
 	print('\t' + __description__)
+
 	print('\t' + '(c)2014 by ' + __author__)
+
 	print('\t' + 'Status:\t' + __status__)
+
 	print('\t' + 'For legal purposes only!')
+
 	print(70*'*' + '\n')
+
 	sys.exit( not main( sys.argv ) )
+