Update typoenum.py

typoenum.py

@@ -9,10 +9,11 @@
 ## v0.5 Added support for Typo v6.X  	     ##
 ## v0.6 Added extension search		     ##
 ## v0.7 Added version search for extensions  ##
+## v0.8 Added support for TOR Service 	     ##
 ###############################################
 
 ############ Version information ##############
-__version__ = "0.7"
+__version__ = "0.8"
 __program__ = "Typo-Enumerator v" + __version__
 __description__ = 'Find out the Typo3 Version, Login-URL and Extensions'
 __author__ = "Jan Rude"
@@ -30,7 +31,6 @@ import urllib
 import urllib2
 import argparse
 import datetime
-import requests
 from Queue import Queue
 from colorama import Fore
 from os.path import isfile
@@ -49,8 +49,9 @@ verbosity = False
 # Checks the Typo version
 def check_typo_version_ChangeLog(domain):
 	global verbosity
+	global user_agent
 	try:
-		url = 'http://' + domain + '/typo3_src/ChangeLog'
+		url = urllib2.Request('http://' + domain + '/typo3_src/ChangeLog', None, user_agent)
 		f = urllib2.urlopen(url, timeout = 3.0)
 		changelog = f.read(200)
 		f.close()
@@ -66,8 +67,9 @@ def check_typo_version_ChangeLog(domain):
 
 def check_typo_version_NEWS_TXT(domain):
 	global verbosity
+	global user_agent
 	try:
-		url = 'http://' + domain + '/typo3_src/NEWS.txt'
+		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.txt', None, user_agent)
 		f = urllib2.urlopen(url, timeout = 3.0)
 		changelog = f.read(500)
 		f.close()
@@ -82,8 +84,9 @@ def check_typo_version_NEWS_TXT(domain):
 		check_typo_version_NEWS_MD(domain)
 
 def check_typo_version_NEWS_MD(domain):
+	global user_agent
 	try:
-		url = 'http://' + domain + '/typo3_src/NEWS.md'
+		url = urllib2.Request('http://' + domain + '/typo3_src/NEWS.md', None, user_agent)
 		f = urllib2.urlopen(url, timeout = 3.0)
 		changelog = f.read(80)
 		f.close()
@@ -99,38 +102,22 @@ def check_typo_version_NEWS_MD(domain):
 def check_typo_login(domain):
 	global user_agent
 	try:	
-		r = requests.get('http://' + domain + '/typo3/index.php', allow_redirects=False, timeout=3.0, headers=user_agent)
+		req = urllib2.Request('http://' + domain + '/typo3/index.php', None, user_agent)
-		statusCode = r.status_code
+		connection = urllib2.urlopen(req)
-		httpResponse = r.text
+		response = connection.read()
-		r.close()
+		return check_title(response, connection.geturl())
-		if statusCode == 200:
+		connection.close()
-			return check_title(httpResponse, r.url)
+	except urllib2.HTTPError, e:
-
+		if e.code == 403:
-		elif (statusCode == 301) or (statusCode == 302):
+			return check_title(response, connection.geturl())
-			location = r.headers['location']
+		elif e.code == 404:
-		 	if verbosity:
-					print "Redirect to:".ljust(32) + location
-		 	if ("http://") in location:
-		 		new = location.split("//")
-		 		new2 = new[1].split("/")
-		 		check_typo_login(new2[0])
-		 	elif ("https://") in location:
-				r = requests.get(location, timeout=3.0, headers=user_agent, verify=False)
-				statusCode = r.status_code
-				httpResponse = r.text
-		 		return check_title(httpResponse, r.url)
-		elif statusCode == 404:
 			print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
+	except urllib2.URLError, e:
+		print str(e.reason)
+	except Exception, e:
+		import traceback
+		print ('generic exception: ', traceback.format_exc())
 
-		else:
-			print "Oops! Got:".ljust(32) + str(statusCode) + ": " + str(r.raise_for_status())
-
-	except requests.exceptions.Timeout:
-		print Fore.RED + "Connection timed out" + Fore.RESET
-	except requests.exceptions.TooManyRedirects:
-		print Fore.RED + "Too many redirects" + Fore.RESET
-	except requests.exceptions.RequestException as e:
-		print Fore.RED + str(e) + Fore.RESET
 
 # Checks, if URL is a Typo-Login
 def check_title(httpResponse, url):
@@ -151,38 +138,48 @@ def check_extensions(domain, input_queue, output_queue):
 	while True:
 		extension = input_queue.get()
 		try:
-			r = requests.get('http://' + domain + '/typo3conf/ext/' + extension + "/", allow_redirects=False, timeout=8.0, headers=user_agent)
+			req = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + "/", None, user_agent)
-			statusCode = r.status_code
+			connection = urllib2.urlopen(req)
-			r.close()
+			connection.close()
-			if (statusCode == 200) or (statusCode == 403):
+			check_extension_version(domain, extension, output_queue)
-				check_extension_version(domain, extension, output_queue) 
+		except urllib2.HTTPError, e:
-			elif statusCode == 404:
+			if e.code == 403:
+				check_extension_version(domain, extension, output_queue)
+			elif e.code == 404:
 				if verbosity:
 					output_queue.put(extension.ljust(32) + Fore.RED + "not installed" + Fore.RESET)
 				pass
-			else:
+		except urllib2.URLError, e:
-				print "Oops! Got:".ljust(32) + str(statusCode) + ": " + str(r.raise_for_status())
+			print str(e.reason)
 		except Exception, e:
-			print "Oops! Got:".ljust(32), e
+				import traceback
+				print ('generic exception: ', traceback.format_exc())
 		input_queue.task_done()
 
 # Searches for version of installed extension
 def check_extension_version(domain, extension, output_queue):
 	global verbosity
+	global user_agent
 	try:
-		url = 'http://' + domain + '/typo3conf/ext/' + extension + '/ChangeLog'
+		url = urllib2.Request('http://' + domain + '/typo3conf/ext/' + extension + '/ChangeLog', None, user_agent)
-		f = urllib2.urlopen(url, timeout = 3.0)
+		connection = urllib2.urlopen(url, timeout = 15.0)
-		changelog = f.read(500)
+		changelog = connection.read(1500)
-		f.close()
+		connection.close()
-		regex = re.compile("\* (.*)")
+		regex = re.compile("(\d{1,2}\.\d{1,2}\.[0-9][0-9]?[' '\n])")
 		searchVersion = regex.search(changelog)
 		version = searchVersion.groups()
-		output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (" + version[0] + ")" + Fore.RESET)
+		output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (v" + version[0].split()[0] + ")" + Fore.RESET)
 	except:
-		if verbosity:
+		try:
-			output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (could not find version)")
+			regex = re.compile("(\d{2,4}[\.\-]\d{1,2}[\.\-]\d{1,4})")
-		else:
+			searchVersion = regex.search(changelog)
-			output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)
+			version = searchVersion.groups()
+			output_queue.put(extension.ljust(32) + Fore.GREEN + "installed (last entry from " + version[0] + ")" + Fore.RESET)
+		except:
+			if verbosity:
+				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (could not find version)")
+			else:
+				output_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)
 
 # Output
 def output_thread(q):
@@ -201,7 +198,7 @@ def output_thread(q):
 def generate_extensions_list(top):
 	global extension_list
 	extension = 'extensions.xml'
-	print "Loading extensions..."
+	print "\nLoading extensions..."
 	if not isfile(extension):
 	 	print(Fore.RED + "File not found: " + extension + "\nAborting..." + Fore.RESET)
 	 	sys.exit(-2)
@@ -213,14 +210,13 @@ def generate_extensions_list(top):
 	for extensions in tag_dict.getchildren():
 			ext = {extensions.get('extensionkey'):extensions[0].text}
 			exten_Dict.update(ext)
-	print 'Loaded ' , len(exten_Dict), ' extensions'
+	print 'Loaded ' , len(exten_Dict), ' extensions\n'
 
 	if top is not None:
 		sorted_dict = sorted(exten_Dict.iteritems(), key=lambda x: int(x[1]), reverse=True)
 			
 		for i in xrange(0,top):
 			extension_list.append(sorted_dict[i][0])
-
 	else:
 		for extension_name in tag_dict:
 			extension_list.append(extension_name.get('extensionkey'))
@@ -231,7 +227,7 @@ def copy_extensions(input_queue):
 	for ext in extension_list:
 		input_queue.put(ext)
 
-# Processbar
+# Progressbar
 def dlProgress(count, blockSize, totalSize):
 	percent = int(count*blockSize*100/totalSize)
 	sys.stdout.write("\rDownloading extentions: " + "%d%%" % percent)
@@ -239,7 +235,6 @@ def dlProgress(count, blockSize, totalSize):
 
 # Update function
 def update():
-	global user_agent
 	try:
 		urllib.urlretrieve('http://ter.sitedesign.dk/ter/extensions.xml.gz', 'extensions.gz', reporthook=dlProgress)
 		inf = gzip.open('extensions.gz', 'rb')
@@ -253,8 +248,36 @@ def update():
 	except Exception, e:
 		print "Oops! Got:".ljust(32), e
 
-#Starting checks
+# Using Privoxy and TOR for all connections
-def start(domain):
+def setting_up_tor():
+	try:
+		import socks
+	except:
+		print "The module 'SocksiPy' is not installed.\nPlease install it with: sudo apt-get install python-socksipy"
+		sys.exit(-2)
+
+	print "Setting up proxy to Privoxy on Port 8118"
+	socks.setdefaultproxy(socks.PROXY_TYPE_HTTP, "127.0.0.1", 8118, True)
+	socket.socket = socks.socksocket
+	try:
+		url = urllib2.Request('https://check.torproject.org/')
+		torcheck = urllib2.urlopen(url)
+		response = torcheck.read()
+		torcheck.close()
+	except:
+		print "Failed to connect to Privoxy and/or TOR!\nPlease make sure they are running and configured!\nYou can start them with:\nservice privoxy start\nservice tor start\n"
+		sys.exit(-2)
+	try:
+		regex = re.compile('Congratulations. This browser is configured to use Tor.')
+		searchVersion = regex.search(response)
+		version = searchVersion.groups()
+		print "Connection to TOR established"
+	except:
+		print "It seems like TOR is not used.\nAborting...\n"
+		sys.exit(-2)
+
+# Starting checks
+def start(domain, top, tor):
 	in_queue = Queue()
 	out_queue = Queue()
 	regex = re.compile("(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
@@ -267,10 +290,17 @@ def start(domain):
 		print("\n\n[*] Check for " + domain)		
 		
 	if check_typo_login(domain) is True:
+		if not extension_list:
+			generate_extensions_list(top)
+		
 		check_typo_version_ChangeLog(domain)
 		copy_extensions(in_queue)
-		print '\nChecking', in_queue.qsize(), 'Extensions:'
+		print '\nChecking', in_queue.qsize(), 'Extensions:\nThis may take a while...'
-		for i in xrange(0, 10):
+		if tor:
+			threads = 2
+		else:
+			threads = 10
+		for i in xrange(0, threads):
 			t = Thread(target=check_extensions, args=(domain, in_queue, out_queue))
 			t.daemon = True
 			t.start()
@@ -285,7 +315,7 @@ def start(domain):
 def main(argv):
 	global user_agent
 	global verbosity
-	parser = argparse.ArgumentParser(add_help=False, usage='typoenum.py -d DOMAIN [DOMAIN ...] | -f FILE [--user_agent USER-AGENT] [--top VALUE] [-v]')
+	parser = argparse.ArgumentParser(add_help=False, usage='typoenum.py -d DOMAIN [DOMAIN ...] | -f FILE [--user_agent USER-AGENT] [--top VALUE] [-v] [--tor]')
 	group = parser.add_mutually_exclusive_group()
 	group.add_argument('-d', '--domain', dest='domain', type=str, nargs='+')
 	group.add_argument('-f', '--file', dest='file', help='File with a list of domains')
@@ -293,23 +323,26 @@ def main(argv):
 	parser.add_argument('--user_agent', dest='user_agent', default='Mozilla/5.0', metavar='USER-AGENT (default: Mozilla/5.0)')
 	parser.add_argument('--top', type=int, dest='top', metavar='VALUE', help='Check only most X downloaded extensions (default: all)', default=None)
 	parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
+	parser.add_argument("--tor", help="using tor for connections", action="store_true")
 	args = parser.parse_args()
 	
 	if not args.domain and not args.file and not args.update:
 		parser.print_help()
 		return True
 
+	if args.tor:
+		setting_up_tor()
+
 	if args.update:
 		update()
 		return True
 
 	user_agent = {'User-Agent' : args.user_agent}
-	generate_extensions_list(args.top)
 	verbosity = args.verbose
 
 	if args.domain and not args.file:
 		for dom in args.domain:
-			start(dom)
+			start(dom, args.top, args.tor)
 
 	elif not args.domain and args.file:
 		if not isfile(args.file):
@@ -318,7 +351,7 @@ def main(argv):
 		else:
 			with open(args.file, 'r') as f:
 				for line in f:
-					start(line.strip())
+					start(line.strip(), args.top, args.tor)
 	print '\n'
 	now = datetime.datetime.now()
 	print __program__ + ' finished at ' + now.strftime("%Y-%m-%d %H:%M:%S") + '\n'