Update to v0.3.3

lib/extensions.py

@@ -5,27 +5,37 @@ Copyright (c) 2014 Jan Rude
 """
 
 import re
+import os
+import sys
 import time
 import urllib2
 from Queue import Queue
-from colorama import Fore
+try:
+	from colorama import Fore
+except:
+	pass
 from os.path import isfile
 from threading import Thread, Lock
 from lib import settings
 
 def generate_list():
-	if not isfile('extensions'):
-		print(Fore.RED + "\nExtensionfile not found!\nPlease update Typo-Enumerator (python typoenum.py -u)" + Fore.RESET)
-		sys.exit(-2)
-	with open('extensions', 'r') as f:
-		count = 0
-		for extension in f:
-			if settings.TOP_EXTENSION > count:
-				settings.EXTENSION_LIST.append(extension.split('\n')[0])
-				count += 1
-			else:
-				f.close()
-				return
+	print ''
+	for ext_file in settings.EXTENSION_FILE:
+		if not isfile(os.path.join('extensions', ext_file)):
+			output("Could not find extension file " + ext_file 
+					+ "\nPossible values are: experimental | alpha | beta | stable | outdated | all", False)
+			sys.exit(-1)
+
+		with open(os.path.join('extensions', ext_file), 'r') as f:
+			count = 0
+			print "[+] Loading:", ext_file
+			for extension in f:
+				if settings.TOP_EXTENSION > count:
+					settings.EXTENSION_LIST.append(extension.split('\n')[0])
+					count += 1
+				else:
+					f.close()
+					return
 
 def copy():
 	for extension in settings.EXTENSION_LIST:
@@ -56,17 +66,18 @@ def check_extension():
 				#	settings.in_queue.put(extension)
 		# if extension is not in any given path, it's not installed
 		if settings.verbose:
-			settings.out_queue.put(extension.ljust(32) + Fore.RED + 'not installed' + Fore.RESET)
+			output(extension.ljust(32) + 'not installed', False)
 		settings.in_queue.task_done()
 
 # Searching version of installed extension
 def check_extension_version(path, extension):
+	settings.EXTENSIONS_FOUND += 1
 	# if no version information is available, skip version search
 	if extension in settings.NO_VERSIONINFO:
 		if settings.verbose:
-			settings.out_queue.put(extension.ljust(32) + Fore.GREEN + 'installed' + Fore.RESET + ' (no version information available)')
+			output(extension.ljust(32) + 'installed (no version information available)', True)
 		else:
-			settings.out_queue.put(extension.ljust(32) + Fore.GREEN + 'installed' + Fore.RESET)
+			output(extension.ljust(32) + 'installed', True)
 	else:
 		try:
 			request = urllib2.Request(settings.DOMAIN + path + extension +'/ChangeLog', None, settings.user_agent)
@@ -77,17 +88,26 @@ def check_extension_version(path, extension):
 				regex = re.compile("(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?[' '\n])")
 				searchVersion = regex.search(changelog)
 				version = searchVersion.groups()
-				settings.out_queue.put(extension.ljust(32) + Fore.GREEN + 'installed (v' + version[0].split()[0] + ')' + Fore.RESET)
+				output(extension.ljust(32) + 'installed (v' + version[0].split()[0] + ')', True)
 			except:
 				try:
 					regex = re.compile("(\d{2,4}[\.\-]\d{1,2}[\.\-]\d{1,4})")
 					search = regex.search(changelog)
 					version = search.groups()
-					settings.out_queue.put(extension.ljust(32) + Fore.GREEN + 'installed (last entry from ' + version[0] + ')' + Fore.RESET)
+					output(extension.ljust(32) + 'installed (last entry from ' + version[0] + ')', True)
 				except:
 					if settings.verbose:
-						settings.out_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET + " (no version information found)")
+						output(extension.ljust(32) + 'installed (no version information found)', True)
 					else:
-						settings.out_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)
+						output(extension.ljust(32) + 'installed', True)
 		except:
-			settings.out_queue.put(extension.ljust(32) + Fore.GREEN + "installed" + Fore.RESET)
+			output(extension.ljust(32) + "installed", True)
+
+def output(message, status):
+	if settings.COLORAMA:
+		if status:
+			print Fore.GREEN + message + Fore.RESET
+		else:
+			print Fore.RED + message + Fore.RESET
+	else:
+		print message

lib/login.py

@@ -40,11 +40,11 @@ def search_login():
 		else:
 			print 'Oops! Got unhandled code:'.ljust(32) + str(statusCode) + ': ' + str(r.raise_for_status())
 	except requests.exceptions.Timeout:
-		print Fore.RED + 'Connection timed out' + Fore.RESET
+		output('Connection timed out')
 	except requests.exceptions.TooManyRedirects:
-		print Fore.RED + 'Too many redirects' + Fore.RESET
+		output('Too many redirects')
 	except requests.exceptions.RequestException as e:
-		print Fore.RED + str(e) + Fore.RESET
+		output(str(e))
 
 # Searching for Typo3 references in title
 def check_title(response, url):
@@ -72,7 +72,7 @@ def check_main_page():
 			if 'fe_typo_user' in cookie:
 				return bad_url()
 		except KeyboardInterrupt:
-			print Fore.RED + '\nReceived keyboard interrupt.\nQuitting...' + Fore.RESET
+			output('\nReceived keyboard interrupt.\nQuitting...')
 			exit(-1)
 		except:
 			try:
@@ -91,14 +91,14 @@ def check_main_page():
 				pass
 	except Exception, e:
 		if '404' in str(e):
-			print Fore.RED + str(e) + '\nPlease ensure you entered the right url' + Fore.RESET
+			output(str(e) + '\nPlease ensure you entered the right url')
 		else:
-			print Fore.RED + str(e) + Fore.RESET
+			output(str(e))
 		return 'skip'
 	return False
 
 def bad_url():
-	print 'Typo3 Login:'.ljust(32) + Fore.GREEN + 'Typo3 is used, but could not find login' + Fore.RESET
+	print 'Typo3 Login:'.ljust(32) + 'Typo3 is used, but could not find login'
 	print ''.ljust(32) + 'This could result in \'no extensions are installed\'.'
 	print ''.ljust(32) + 'Seems like something is wrong with the given url.'
 	answer = ''
@@ -110,3 +110,11 @@ def bad_url():
 	if answer is 'y':
 		return True
 	return 'skip'
+
+# printing error messages
+def output(message):
+	if settings.COLORAMA:
+		print Fore.RED
+	print message
+	if settings.COLORAMA:
+		print Fore.RESET

lib/privoxy_only.py

@@ -2,7 +2,7 @@ import socket
 import urllib2
 import os, sys
 import re
-from colorama import Fore
+
 try:
 	import socks
 except:
@@ -16,7 +16,7 @@ except:
 def start_daemon():
 	if sys.platform.startswith('linux'):
 		os.system('service privoxy start')
-		print '[' + Fore.GREEN + ' ok ' + Fore.RESET + '] Starting privoxy daemon...done.'
+		print '[ ok ] Starting privoxy daemon...done.'
 	elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
 		print "Please make sure Privoxy is running..."
 	else:
@@ -34,7 +34,7 @@ def connect(port):
 		response = torcheck.read()
 		torcheck.close()
 	except:
-		print Fore.RED + "Failed to connect through Privoxy!" + Fore.RESET
+		print "Failed to connect through Privoxy!"
 		print "Please make sure your configuration is right!\n"
 		sys.exit(-2)
 	try:
@@ -51,6 +51,6 @@ def stop():
 	print "\n"
 	if sys.platform.startswith('linux'):
 		os.system('service privoxy stop')
-		print '[' + Fore.GREEN + ' ok ' + Fore.RESET + '] Stopping privoxy daemon...done.'
+		print '[ ok ] Stopping privoxy daemon...done.'
 	elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
-		print "You can close Privoxy now..."
+		print "You can close Privoxy now..."

lib/settings.py

@@ -8,8 +8,7 @@ from Queue import Queue
 from threading import Thread, Lock
 
 # Domain to check
-# Valid: string
-DOMAIN = ""
+DOMAIN = ''
 
 # Maximum number of threads (avoiding connection issues and/or DoS)
 MAX_NUMBER_OF_THREADS = 10
@@ -20,6 +19,10 @@ DEFAULT_TOR_PORT = 9050
 # Default ports used in Tor proxy bundles
 DEFAULT_PRIVOXY_PORT = 8118
 
+# Default extension file
+# Default: all available Typo3 extensions
+EXTENSION_FILE = ['all_extensions']
+
 # List with selected extensions
 EXTENSION_LIST = []
 
@@ -27,7 +30,7 @@ EXTENSION_LIST = []
 NO_VERSIONINFO = ['wt_spamshield', 'introduction'] #introduction has ChangeLog, but will use "Typo3 4.5.0" as version info!
 
 # Check only top X extensions
-# Default: all
+# Default: all extensions
 TOP_EXTENSION = 'all'
 
 # HTTP User-Agent header value. Useful to fake the HTTP User-Agent header value at each HTTP request
@@ -35,19 +38,16 @@ TOP_EXTENSION = 'all'
 user_agent = {'User-Agent' : "Mozilla/5.0"}
 
 # Maximum number of concurrent HTTP(s) requests (handled with Python threads)
-# Valid: integer
 # Default: 7
 THREADS = 7
 
 # Verbosity.
 verbose = False
 
-#Input and output queues
-in_queue = ""
-out_queue = ""
+#Input queue
+in_queue = ''
 
 # Seconds to wait before timeout connection.
-# Valid: int
 # Default: 20
 TIMEOUT = 20
 
@@ -67,8 +67,12 @@ EXTENSION_PATHS = ('/typo3conf/ext/', '/typo3/sysext/')
 # Possible version info file
 EXTENSION_VERSION_INFO = ('ChangeLog', 'README.txt')
 
+# Installed extensions on targed domain
 EXTENSIONS_FOUND = 0
 
+# Colorama is not used, if not installed
+COLORAMA = False
+
 
 
 ## Not used atm ##

lib/start.py

@@ -8,7 +8,10 @@ import time
 from Queue import Queue
 from os.path import isfile
 from threading import Thread, Lock
-from colorama import Fore, Back
+try:
+	from colorama import Fore, Back
+except:
+	pass
 from lib import settings
 from lib import versioninfo
 from lib import login
@@ -18,7 +21,12 @@ from lib import extensions
 # Startmethod
 def check_typo_installation(domain):
 	settings.DOMAIN = domain
-	print '\n\n' + Fore.CYAN + '[ Checking ' + domain + ' ]' + '\n' + "-"* 70  + Fore.RESET
+	settings.EXTENSIONS_FOUND = 0
+	if settings.COLORAMA:
+		output = Fore.CYAN + '[ Checking ' + domain + ' ]' + '\n' + "-"* 70  + Fore.RESET
+	else:
+		output = '[ Checking ' + domain + ' ]' + '\n' + "-"* 70
+	print '\n\n' + output
 
 	check = login.search_login()
 	if check is "redirect":
@@ -31,21 +39,20 @@ def check_typo_installation(domain):
 		if mainpage is True:
 			init_extension_search()
 		elif mainpage is not "skip":
-			print "Typo3 Login:".ljust(32) + Fore.RED + "Typo3 is not used on this domain" + Fore.RESET
+			output("Typo3 Login:".ljust(32) + "Typo3 is not used on this domain", False)
 
 def init_extension_search():
 	settings.in_queue = Queue()
-	settings.out_queue = Queue()
 	versioninfo.search_version_info()
 	versioninfo.output()
 
-	if not settings.EXTENSION_LIST:
-		extensions.generate_list()
+	if settings.TOP_EXTENSION != 0:
+		if not settings.EXTENSION_LIST:
+			extensions.generate_list()
 
-	extensions.copy()
-	extensions_to_check = settings.in_queue.qsize()
+		extensions.copy()
+		extensions_to_check = settings.in_queue.qsize()
 
-	if extensions_to_check is not 0:
 		print '\nChecking', extensions_to_check, 'extension(s)...'
 		# Thanks to 'RedSparrow': http://stackoverflow.com/questions/17991033/python-cant-kill-main-thread-with-keyboardinterrupt
 		try:
@@ -59,18 +66,24 @@ def init_extension_search():
 				else:
 					break
 		except KeyboardInterrupt:
-			print Fore.RED + "\nReceived keyboard interrupt.\nQuitting..." + Fore.RESET
+			output("\nReceived keyboard interrupt.\nQuitting...", False)
 			exit(-1)
 		settings.in_queue.join()
 
-		installed_ext = settings.out_queue.qsize()
-		if installed_ext is 0:
-			print Fore.RED + "No extensions installed" + Fore.RESET
+		installed_ext = settings.EXTENSIONS_FOUND
+		if installed_ext == 0:
+			output("No extensions installed", False)
 		else:
-			t = Thread(target=output.thread, args=())
-			t.daemon = True
-			t.start()
-			settings.out_queue.join()
-			print Fore.GREEN + '\n', str(settings.EXTENSIONS_FOUND) + '/' + str(extensions_to_check),'extension(s) installed' + Fore.RESET
+			output('\n' + str(settings.EXTENSIONS_FOUND) + '/' + str(extensions_to_check) + ' extension(s) installed', True)
 	else:
 		print '\nSkipping check for extensions...'
+
+# print error messages
+def output(message, setting):
+	if settings.COLORAMA:
+		if not setting:
+			print Fore.RED + message + Fore.RESET
+		if setting:
+			print Fore.GREEN + message + Fore.RESET
+	else:
+		print message

lib/tor_only.py

@@ -2,7 +2,7 @@ import socket
 import urllib2
 import os, sys
 import re
-from colorama import Fore
+
 try:
 	import socks
 except:
@@ -34,14 +34,14 @@ def connect(port):
 		torcheck.close()
 	except Exception, e:
 		print e
-		print Fore.RED + "Failed to connect through TOR!" + Fore.RESET
+		print "Failed to connect through TOR!"
 		print "Please make sure your configuration is right!\n"
 		sys.exit(-2)
 	try:
 		regex = re.compile('Congratulations. This browser is configured to use Tor.')
 		searchVersion = regex.search(response)
 		version = searchVersion.groups()
-		print Fore.GREEN + "Connection to TOR established" + Fore.RESET
+		print "Connection to TOR established"
 		regex = re.compile("(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
 		searchIP = regex.search(response)
 		IP = searchIP.groups()[0]
@@ -55,4 +55,4 @@ def stop():
 	if sys.platform.startswith('linux'):
 		os.system('service tor stop')
 	elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
-		print "You can close TOR now..."
+		print "You can close TOR now..."

lib/tor_with_privoxy.py

@@ -2,7 +2,7 @@ import socket
 import urllib2
 import os, sys
 import re
-from colorama import Fore
+
 try:
 	import socks
 except:
@@ -17,7 +17,7 @@ def start_daemon():
 	if sys.platform.startswith('linux'):
 		os.system('service tor start')
 		os.system('service privoxy start')
-		print '[' + Fore.GREEN + ' ok ' + Fore.RESET + '] Starting privoxy daemon...done.'
+		print '[ ok ] Starting privoxy daemon...done.'
 	elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
 		print "Please make sure TOR and Privoxy are running..."
 	else:
@@ -35,14 +35,14 @@ def connect(port):
 		response = torcheck.read()
 		torcheck.close()
 	except:
-		print Fore.RED + "Failed to connect through Privoxy and/or TOR!" + Fore.RESET
+		print "Failed to connect through Privoxy and/or TOR!"
 		print "Please make sure your configuration is right!\n"
 		sys.exit(-2)
 	try:
 		regex = re.compile('Congratulations. This browser is configured to use Tor.')
 		searchVersion = regex.search(response)
 		version = searchVersion.groups()
-		print Fore.GREEN + "Connection to TOR established" + Fore.RESET
+		print "Connection to TOR established"
 		regex = re.compile("(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
 		searchIP = regex.search(response)
 		IP = searchIP.groups()[0]
@@ -57,6 +57,6 @@ def stop():
 	if sys.platform.startswith('linux'):
 		os.system('service tor stop')
 		os.system('service privoxy stop')
-		print '[' + Fore.GREEN + ' ok ' + Fore.RESET + '] Stopping privoxy daemon...done.'
+		print '[ ok ] Stopping privoxy daemon...done.'
 	elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
-		print "You can close TOR and Privoxy now..."
+		print "You can close TOR and Privoxy now..."

lib/update.py

@@ -8,12 +8,13 @@ import xml.etree.ElementTree as ElementTree
 # Progressbar
 def dlProgress(count, blockSize, totalSize):
 	percent = int(count*blockSize*100/totalSize)
-	sys.stdout.write("\rDownloading extentions: " + "%d%%" % percent)
+	sys.stdout.write("\r[+] Downloading extentions: " + "%d%%" % percent)
 	sys.stdout.flush()
 
 # Download extensions from typo3 repository
 def download_ext():
 	try:
+		urllib.URLopener.version = 'TYPO3/7.0.0'		
 		urllib.urlretrieve('http://ter.sitedesign.dk/ter/extensions.xml.gz', 'extensions.gz', reporthook=dlProgress)
 		inf = gzip.open('extensions.gz', 'rb')
 		file_content = inf.read()
@@ -23,22 +24,82 @@ def download_ext():
 		outF.close()
 		os.remove('extensions.gz')
 	except Exception, e:
-		print "Oops! Got:".ljust(32), e
+		print "\nOops! Got:", e
 
-# Parse extensions.xml and save extensions in file
+# Parse extensions.xml and save extensions in files
 def generate_list():
-	extension = 'extensions.xml'
-	print "\nParsing file..."
-	tree = ElementTree.parse(extension) 
-	tag_dict = tree.getroot()
-	exten_Dict = {}
-	for extensions in tag_dict.getchildren():
-			ext = {extensions.get('extensionkey'):extensions[0].text}
-			exten_Dict.update(ext)	
-	sorted_dict = sorted(exten_Dict.iteritems(), key=lambda x: int(x[1]), reverse=True)	
-	f = open('extensions','w')
-	for i in xrange(0,len(exten_Dict)):
-		f.write(sorted_dict[i][0]+'\n')
+	experimental = {} # everything with 'experimental' and 'test'
+	alpha = {}
+	beta = {}
+	stable = {}
+	outdated = {} # everything with 'obsolete' and 'outdated'
+	allExt = {}
+
+	print "\n[+] Parsing file..."
+	tree = ElementTree.parse('extensions.xml') 
+	root = tree.getroot()
+	extension = 0
+	# for every extension in file
+	for child in root:
+		# insert every extension in "allExt" dictionary
+		allExt.update({child.get('extensionkey'):child[0].text})
+		# and search the last version entry
+		version = 0
+		for version_entry in root[extension].iter('version'):
+			version +=1
+		# get the state of the latest version
+		state = (str(root[extension][version][2].text)).lower()
+		if state == 'experimental' or state == 'test':
+			experimental.update({child.get('extensionkey'):child[0].text})
+		elif state == 'alpha':
+			alpha.update({child.get('extensionkey'):child[0].text})
+		elif state == 'beta':
+			beta.update({child.get('extensionkey'):child[0].text})
+		elif state == 'stable':
+			stable.update({child.get('extensionkey'):child[0].text})
+		elif state == 'obsolete' or state == 'outdated':
+			outdated.update({child.get('extensionkey'):child[0].text})
+		extension+=1
+
+	# sorting lists according to number of downloads
+	print "[+] Sorting according to number of downloads..."
+	sorted_experimental = sorted(experimental.iteritems(), key=lambda x: int(x[1]), reverse=True)
+	sorted_alpha = sorted(alpha.iteritems(), key=lambda x: int(x[1]), reverse=True)
+	sorted_beta = sorted(beta.iteritems(), key=lambda x: int(x[1]), reverse=True)
+	sorted_stable = sorted(stable.iteritems(), key=lambda x: int(x[1]), reverse=True)
+	sorted_outdated = sorted(outdated.iteritems(), key=lambda x: int(x[1]), reverse=True)
+	sorted_allExt = sorted(allExt.iteritems(), key=lambda x: int(x[1]), reverse=True)
+
+	print "[+] Generating files..."
+	f = open(os.path.join('extensions', 'experimental_extensions'),'w')
+	for i in xrange(0,len(sorted_experimental)):
+		f.write(sorted_experimental[i][0]+'\n')
 	f.close()
-	print 'Loaded', len(exten_Dict), 'extensions\n'
+
+	f = open(os.path.join('extensions', 'alpha_extensions'),'w')
+	for i in xrange(0,len(sorted_alpha)):
+		f.write(sorted_alpha[i][0]+'\n')
+	f.close()
+
+	f = open(os.path.join('extensions', 'beta_extensions'),'w')
+	for i in xrange(0,len(sorted_beta)):
+		f.write(sorted_beta[i][0]+'\n')
+	f.close()
+
+	f = open(os.path.join('extensions', 'stable_extensions'),'w')
+	for i in xrange(0,len(sorted_stable)):
+		f.write(sorted_stable[i][0]+'\n')
+	f.close()
+
+	f = open(os.path.join('extensions', 'outdated_extensions'),'w')
+	for i in xrange(0,len(sorted_outdated)):
+		f.write(sorted_outdated[i][0]+'\n')
+	f.close()
+
+	f = open(os.path.join('extensions', 'all_extensions'),'w')
+	for i in xrange(0,len(sorted_allExt)):
+		f.write(sorted_allExt[i][0]+'\n')
+	f.close()
+
+	print '[+] Loaded', len(sorted_allExt), 'extensions\n'
 	os.remove('extensions.xml')

lib/versioninfo.py

@@ -6,8 +6,11 @@ Copyright (c) 2014 Jan Rude
 
 import re
 import urllib2
-from colorama import Fore
 from lib import settings
+try:
+	from colorama import Fore
+except:
+	pass
 
 # Searching for Typo3 version
 def search_version_info():
@@ -29,7 +32,16 @@ def search_version_info():
 # Output of Typo3 version
 def output():
 	if settings.TYPO_VERSION is None:
-		print "Typo3 Version:".ljust(32) + Fore.RED + "Not found" + Fore.RESET
+		print "Typo3 Version:".ljust(32)
+		if settings.COLORAMA:
+			print Fore.RED 
+		print "Not found"
+		if settings.COLORAMA:
+			Fore.RESET
 	else:
-		print "Typo3 Version:".ljust(32) + Fore.GREEN + settings.TYPO_VERSION + Fore.RESET
+		if settings.COLORAMA:
+			output = Fore.GREEN + settings.TYPO_VERSION + Fore.RESET
+		else:
+			output = settings.TYPO_VERSION
+		print "Typo3 Version:".ljust(32) + output
 		print "Link to vulnerabilities:".ljust(32) + "http://www.cvedetails.com/version-search.php?vendor=&product=Typo3&version=" + settings.TYPO_VERSION.split()[1]