fixed agent bug, cosmetic, added output url of found plugins

fix error caused by unknown extension

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

doc/CHANGELOG.md

@@ -1,3 +1,18 @@
+## Version 0.6.3
+
+* Fixed advisory URLs
+* Fixed rootCheck
+
+## Version 0.6.2
+
+* Bugfix in extension vulnerability parsing
+* Bugfix on database reset
+
+## Version 0.6.1
+
+* Bugfix of URL determination
+* Support for JSON output
+
 ## Version 0.6
 
 * Added version regex for composer installations

lib/config.json

@@ -1 +1 @@
-{"threads": 5, "timeout": 10, "cookie": "", "auth": "", "User-Agent": "Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0"}
+{"threads": 3, "timeout": 10, "cookie": "", "auth": "", "User-Agent": "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16"}

lib/domain.py

@@ -29,9 +29,11 @@ class Domain:
     """
     This class stores following information about a domain:
         name:                   URL of the domain
-        typo3:                  If Typo3 is installed
-        typo3_version:          Typo3 Version
         path:                   Full path to Typo3 installation
+        typo3:                  If Typo3 is installed
+        backend:                URL to Typo3 backend login
+        typo3_version:          Typo3 Version
+        typo3_vulnerabilities:  List of known CORE vulnerabilities
         installed_extensions:   List of all installed extensions
     """
     def __init__(self, name):
@@ -39,10 +41,12 @@ class Domain:
             self.__name = 'https://' + name
         else:
             self.__name = name
-        self.__typo3 = False
-        self.__typo3_version = ''
         self.__path = ''
-        self.__installed_extensions = {}
+        self.__typo3 = False
+        self.__backend = 'Could not be found'
+        self.__typo3_version = 'Unknown'
+        self.__typo3_vulnerabilities = ''
+        self.__installed_extensions = {'installed': 0}
         
     def get_name(self):
         return self.__name
@@ -50,23 +54,35 @@ class Domain:
     def set_name(self, name):
         self.__name = name
 
+    def set_path(self, path):
+        self.__path = path
+
+    def get_path(self):
+        return self.__path
+
     def is_typo3(self):
         return self.__typo3
 
     def set_typo3(self):
         self.__typo3 = True
 
+    def set_backend(self, url):
+        self.__backend = url
+
+    def get_backend(self):
+        return self.__backend
+
     def set_typo3_version(self, version):
         self.__typo3_version = version
 
     def get_typo3_version(self):
         return self.__typo3_version
 
-    def set_path(self, path):
+    def set_typo3_vulns(self, vuln):
-        self.__path = path
+        self.__typo3_vulnerabilities = vuln
 
-    def get_path(self):
+    def get_typo3_vulns(self):
-        return self.__path
+        return self.__typo3_vulnerabilities
 
     def check_root(self):
         """
@@ -75,17 +91,13 @@ class Domain:
             If found, it searches for a Typo3 path reference
             in order to determine the Typo3 installation path.
         """
-        response = request.get_request('{}'.format(self.get_name()))
         full_path = self.get_name()
+        response = request.get_request('{}'.format(self.get_name()))
         if re.search('powered by TYPO3', response['html']):
             self.set_typo3()
-            path = re.search('="/?(\S*?)/?(?:typo3temp|typo3conf)/'.format(self.get_name()), response['html'])
+            path = re.search('="(?:{})/?(\S*?)/?(?:typo3temp|typo3conf)/'.format(self.get_name()), response['html'])
-            if path and path.groups()[0] != '':
+            if path and path.group(1) != '':
-                path = path.groups()[0].replace(self.get_name(), '')
-                if path != '':
                 full_path = '{}/{}'.format(self.get_name(), path)            
-        if full_path.endswith('/'):
-            full_path = full_path[:-1]
         self.set_path(full_path)
 
     def check_default_files(self):
@@ -131,15 +143,17 @@ class Domain:
             and searches for a specific string in the title or the response.
             If the access is forbidden (403), extension search is still possible.
         """
-        print('[+] Backend Login')
+        print(' [+] Backend Login')
         # maybe /typo3_src/typo3/index.php too?
         response = request.get_request('{}/typo3/index.php'.format(self.get_path()))
         searchTitle = re.search('<title>(.*)</title>', response['html'])
         if searchTitle and 'Login' in searchTitle.group(0):
             print('  \u251c {}'.format(Fore.GREEN + '{}/typo3/index.php'.format(self.get_path()) + Fore.RESET))
+            self.set_backend('{}/typo3/index.php'.format(self.get_path()))
         elif ('Backend access denied: The IP address of your client' in response['html']) or (response['status_code'] == 403):
             print('  \u251c {}'.format(Fore.GREEN + '{}/typo3/index.php'.format(self.get_path()) + Fore.RESET))
             print('  \u251c {}'.format(Fore.YELLOW + 'But access is forbidden (IP Address Restriction)' + Fore.RESET))
+            self.set_backend('{}/typo3/index.php'.format(self.get_path()))
         else:
             print('  \u251c {}'.format(Fore.RED + 'Could not be found' + Fore.RESET))
 
@@ -149,34 +163,35 @@ class Domain:
             The exact version can be found in the ChangeLog, therefore it will be requested first.
             Less specific version information can be found in the NEWS or INSTALL file. 
         """           
-        files = {'/typo3_src/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
+        files = {'typo3_src/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
-                '/typo3_src/public/typo3/sysext/install/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
+                'typo3_src/public/typo3/sysext/install/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
-                '/typo3_src/typo3/sysext/adminpanel/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
+                'typo3_src/typo3/sysext/adminpanel/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"([0-9]+\.[0-9]+\.?[0-9x]?[0-9x]?)"',
-                '/typo3_src/typo3/sysext/backend/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)"',
+                'typo3_src/typo3/sysext/backend/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)"',
-                '/typo3_src/typo3/sysext/info/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)"',
+                'typo3_src/typo3/sysext/info/composer.json': '(?:"typo3/cms-core":|"typo3/cms-backend":)\s?"(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)"',
-                '/typo3_src/ChangeLog': '[Tt][Yy][Pp][Oo]3 (\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
+                'typo3_src/ChangeLog': '[Tt][Yy][Pp][Oo]3 (\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
-                '/ChangeLog': '[Tt][Yy][Pp][Oo]3 (\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
+                'ChangeLog': '[Tt][Yy][Pp][Oo]3 (\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
-                '/typo3/sysext/backend/ext_emconf.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
+                'typo3/sysext/backend/ext_emconf.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
-                '/typo3_src/typo3/sysext/install/Start/Install.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
+                'typo3_src/typo3/sysext/install/Start/Install.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
-                '/typo3/sysext/install/Start/Install.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
+                'typo3/sysext/install/Start/Install.php': '(?:CMS |typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9]?[0-9]?)',
-                '/typo3_src/NEWS.txt': 'http://wiki.typo3.org/TYPO3_(\d{1,2}\.\d{1,2})', 
+                'typo3_src/NEWS.txt': 'http://wiki.typo3.org/TYPO3_(\d{1,2}\.\d{1,2})', 
-                '/typo3_src/NEWS.md': '[Tt][Yy][Pp][Oo]3 [Cc][Mm][Ss] (\d{1,2}\.\d{1,2}) - WHAT\'S NEW',
+                'typo3_src/NEWS.md': '[Tt][Yy][Pp][Oo]3 [Cc][Mm][Ss] (\d{1,2}\.\d{1,2}) - WHAT\'S NEW',
-                '/NEWS.txt': 'http://wiki.typo3.org/TYPO3_(\d{1,2}\.\d{1,2})',
+                'NEWS.txt': 'http://wiki.typo3.org/TYPO3_(\d{1,2}\.\d{1,2})',
-                '/NEWS.md': '[Tt][Yy][Pp][Oo]3 [Cc][Mm][Ss] (\d{1,2}\.\d{1,2}) - WHAT\'S NEW',
+                'NEWS.md': '[Tt][Yy][Pp][Oo]3 [Cc][Mm][Ss] (\d{1,2}\.\d{1,2}) - WHAT\'S NEW',
-                '/typo3_src/INSTALL.md': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
+                'typo3_src/INSTALL.md': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
-                '/typo3_src/INSTALL.txt': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
+                'typo3_src/INSTALL.txt': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
-                '/INSTALL.md': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
+                'INSTALL.md': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)',
-                '/INSTALL.txt': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)'
+                'INSTALL.txt': '(?:typo3_src-)(\d{1,2}\.\d{1,2}\.?[0-9x]?[0-9]?)'
                 }
 
         version = None
+        version_path = None
         for path, regex in files.items():
-            response = request.version_information(self.get_path()+path, regex)
+            response = request.version_information('{}/{}'.format(self.get_path(), path), regex)
             if response and (version is None or (len(response) > len(version))):
                 version = response
                 version_path = path
 
-        print(' |\n[+] Version Information')
+        print('  | \n [+] Version Information')
         if version:
             print('  \u251c Identified Version: '.ljust(28) + '{}'.format(Style.BRIGHT + Fore.GREEN + version + Style.RESET_ALL))
             print('  \u251c Version File: '.ljust(28) + '{}{}'.format(self.get_path(), version_path))
@@ -187,26 +202,28 @@ class Domain:
                     version = version + '.0'
                 else:
                     return False
+            self.set_typo3_version(version)
             # sqlite stuff
             conn = sqlite3.connect('lib/typo3scan.db')
             c = conn.cursor()
             c.execute('SELECT advisory, vulnerability, subcomponent, affected_version_max, affected_version_min FROM core_vulns WHERE (?<=affected_version_max AND ?>=affected_version_min)', (version, version,))
             data = c.fetchall()
-            vuln_list = []
             if data:
+                json_list = {}
                 for vulnerability in data:
                     # maybe instead use this: https://oraerr.com/database/sql/how-to-compare-version-string-x-y-z-in-mysql-2/
                     if parse_version(version) <= parse_version(vulnerability[3]):
-                        vuln_list.append(Style.BRIGHT + '    [!] {}'.format(Fore.RED + vulnerability[0] + Style.RESET_ALL))
+                        json_list[vulnerability[0]] = {'Type': vulnerability[1], 'Subcomponent': vulnerability[2], 'Affected': '{} - {}'.format(vulnerability[3], vulnerability[4]), 'Advisory': 'https://typo3.org/security/advisory/{}'.format(vulnerability[0].lower())}
-                        vuln_list.append('     \u251c Vulnerability Type:'.ljust(28) + vulnerability[1])
+                if json_list:
-                        vuln_list.append('     \u251c Subcomponent:'.ljust(28) + vulnerability[2])
+                    self.set_typo3_vulns(json_list)
-                        vuln_list.append('     \u251c Affected Versions:'.ljust(28) + '{} - {}'.format(vulnerability[3], vulnerability[4]))
-                        vuln_list.append('     \u2514 Advisory URL:'.ljust(28) + 'https://typo3.org/security/advisory/{}\n'.format(vulnerability[0].lower()))
-            if vuln_list:
                     print('  \u2514 Known Vulnerabilities:\n')
-                    for vulnerability in vuln_list:
+                    for vulnerability in json_list.keys():
-                        print(vulnerability)
+                        print(Style.BRIGHT + '     [!] {}'.format(Fore.RED + vulnerability + Style.RESET_ALL))
+                        print('      \u251c Vulnerability Type:'.ljust(28) + json_list[vulnerability]['Type'])
+                        print('      \u251c Subcomponent:'.ljust(28) + json_list[vulnerability]['Subcomponent'])
+                        print('      \u251c Affected Versions:'.ljust(28) + json_list[vulnerability]['Affected'])
+                        print('      \u2514 Advisory URL:'.ljust(28) + json_list[vulnerability]['Advisory'] + '\n')
                 else:
                     print('  \u2514 No Known Vulnerabilities')
         else:
-            print(' \u2514', Fore.RED + 'No Version Information Found.' + Fore.RESET)
+            print('  \u2514', Fore.RED + 'Could not be determined.' + Fore.RESET)

lib/extensions.py

@@ -35,7 +35,6 @@ class Extensions:
         """
             This method loads the extensions from the database and searches for installed extensions.
                 /typo3conf/ext/:        Local installation path. This is where extensions usually get installed.
-                /typo3/ext/:            Global installation path (not used atm)
                 /typo3/sysext/:         Extensions shipped with core
         """
         found_extensions = {}
@@ -43,7 +42,6 @@ class Extensions:
         for ext in extensions:
             thread_pool.add_job((request.head_request, ('{}/typo3conf/ext/{}/'.format(domain, ext))))
             thread_pool.add_job((request.head_request, ('{}/typo3/sysext/{}/'.format(domain, ext))))
-            #thread_pool.add_job((request.head_request, ('{}/typo3/ext/{}/'.format(domain, ext))))
         thread_pool.start(threads)
 
         for installed_extension in thread_pool.get_result():
@@ -94,25 +92,38 @@ class Extensions:
         c = conn.cursor()
         print('\n\n [+] Extension Information')
         print(' -------------------------')
+        json_list = {}
         for extension,info in extension_dict.items():
             c.execute('SELECT title,version,state FROM extensions where extensionkey=?', (extension,))
             data = c.fetchone()
             print(Style.BRIGHT + '  [+] {}'.format(Fore.GREEN + extension + Style.RESET_ALL))
+            if data is None:
+                print('   \u251c Extension ({}) is unknown'.format(extension))
+                continue
             print('   \u251c Extension Title: '.ljust(28) + '{}'.format(data[0]))
             print('   \u251c Extension Repo: '.ljust(28) + 'https://extensions.typo3.org/extension/{}'.format(extension))
+            print('   \u251c Extension Url: '.ljust(28) + '{}'.format(info['url']))
+            if not 'stable' in data[2]:
+            	print('   \u251c Current Version: '.ljust(28) + '{} ({})'.format(data[1], Fore.RED + data[2] + Style.RESET_ALL))
+            else:
             	print('   \u251c Current Version: '.ljust(28) + '{} ({})'.format(data[1], data[2]))
+            json_list[extension] = {'Title': data[0], 'Repo': 'https://extensions.typo3.org/extension/{}'.format(extension), 'Current': '{} ({})'.format(data[1], data[2]), 'Version': '', 'Vulnerabilities':''}
             if info['version']:
+                json_list[extension].update(Version = info['version'])
                 c.execute('SELECT advisory, vulnerability, affected_version_max, affected_version_min FROM extension_vulns WHERE (extensionkey=? AND ?<=affected_version_max AND ?>=affected_version_min)', (extension, info['version'], info['version'],))
                 data = c.fetchall()
                 print('   \u251c Identified Version: '.ljust(28) + '{}'.format(Style.BRIGHT + Fore.GREEN + info['version'] + Style.RESET_ALL))
                 vuln_list = []
                 if data:
+                    vuln = {}
                     for vulnerability in data:
                         if parse_version(info['version']) <= parse_version(vulnerability[2]):
                             vuln_list.append(Style.BRIGHT + '     [!] {}'.format(Fore.RED + vulnerability[0] + Style.RESET_ALL))
                             vuln_list.append('      \u251c Vulnerability Type: '.ljust(28) + vulnerability[1])
                             vuln_list.append('      \u251c Affected Versions: '.ljust(28) + '{} - {}'.format(vulnerability[2], vulnerability[3]))
                             vuln_list.append('      \u2514 Advisory URL:'.ljust(28) + 'https://typo3.org/security/advisory/{}\n'.format(vulnerability[0].lower()))
+                            vuln[vulnerability[0]] = {'Type': vulnerability[1], 'Affected': '{} - {}'.format(vulnerability[2], vulnerability[3]), 'Advisory': 'https://typo3.org/security/advisory/{}'.format(vulnerability[0].lower())}
+                    json_list[extension].update(Vulnerabilities = vuln)
                 if vuln_list:
                     print('   \u251c Version File: '.ljust(28) + '{}'.format(info['file']))
                     print('   \u2514 Known Vulnerabilities:\n')
@@ -124,3 +135,4 @@ class Extensions:
                 print('   \u2514 Identified Version: '.ljust(28) + '-unknown-')
             print()
         conn.close()
+        return json_list

lib/initdb.py

@@ -33,7 +33,7 @@ class DB_Init:
             c.execute('''DROP TABLE IF EXISTS extensions''')
             c.execute('''DROP TABLE IF EXISTS extension_vulns''')
             c.execute('''DROP TABLE IF EXISTS core_vulns''')
-            c.execute('''DROP TABLE IF EXISTS settings''')
+            c.execute('''DROP TABLE IF EXISTS UserAgents''')
             conn.commit()
 
             # Create table extensions
@@ -42,11 +42,11 @@ class DB_Init:
 
             # Create table extension_vulns
             c.execute('''CREATE TABLE IF NOT EXISTS extension_vulns
-                         (advisory text, extensionkey text, vulnerability text, branch_max integer, affected_version_max text, branch_max integer, affected_version_min text)''')
+                         (advisory text, extensionkey text, vulnerability text, affected_version_max text, affected_version_min text)''')
 
             # Create table core_vulns
             c.execute('''CREATE TABLE IF NOT EXISTS core_vulns
-                         (advisory text, vulnerability text, subcomponent text, branch_max integer, affected_version_max text, branch_max integer, affected_version_min text, cve text)''')
+                         (advisory text, vulnerability text, subcomponent text, affected_version_max text, affected_version_min text, cve text)''')
 
             # Create table UserAgents
             c.execute('''CREATE TABLE IF NOT EXISTS UserAgents
@@ -83,7 +83,7 @@ class DB_Init:
             if conn:
                 conn.rollback()
             print(e)
-            sys.exit(-1)
+            exit(-1)
 
         finally:
             if conn:

lib/request.py

@@ -54,6 +54,7 @@ def get_request(url):
         response['html'] = r.text
         response['headers'] = r.headers
         response['cookies'] = r.cookies
+        response['url'] = r.url
         return response
     except requests.exceptions.Timeout as e:
         print(e)

lib/update.py

@@ -57,13 +57,13 @@ class Update:
         """
         print('\n[+] Searching for new CORE vulnerabilities...')
         update_counter = 0
-        response = requests.get('https://typo3.org/help/security-advisories/typo3-cms/1')
+        response = requests.get('https://typo3.org/help/security-advisories/typo3-cms/page-1')
-        pages = re.findall('<a class=\"page-link\" href=\"/help/security-advisories/typo3-cms/([0-9]+)\">', response.text)
+        pages = re.findall('<a class=\"page-link\" href=\"/help/security-advisories/typo3-cms/page-([0-9]+)\">', response.text)
         last_page = int(pages[-1])
 
         for current_page in range(1, last_page+1):
             print(' \u251c Page {}/{}'.format(current_page, last_page))
-            response = requests.get('https://typo3.org/help/security-advisories/typo3-cms/{}'.format(current_page), timeout=6)
+            response = requests.get('https://typo3.org/help/security-advisories/typo3-cms/page-{}'.format(current_page), timeout=6)
             advisories = re.findall('TYPO3-CORE-SA-[0-9][0-9][0-9][0-9]-[0-9][0-9][0-9]', response.text)
             for advisory in advisories:
                 vulnerabilities = []
@@ -153,19 +153,18 @@ class Update:
                     exit(-1)
 
                 # Add vulnerability details to database
-                for ext_vuln in vulnerabilities:
+                for core_vuln in vulnerabilities:
-                    c.execute('SELECT * FROM core_vulns WHERE advisory=? AND vulnerability=? AND subcomponent=? AND affected_version_max=? AND affected_version_min=? AND cve=?', (ext_vuln[0], ext_vuln[1], ext_vuln[2], ext_vuln[3], ext_vuln[4], ext_vuln[5],))
+                    c.execute('SELECT * FROM core_vulns WHERE advisory=? AND vulnerability=? AND subcomponent=? AND affected_version_max=? AND affected_version_min=? AND cve=?', (core_vuln[0], core_vuln[1], core_vuln[2], core_vuln[3], core_vuln[4], core_vuln[5],))
                     data = c.fetchall()
                     if not data:
                         update_counter+=1
-                        c.execute('INSERT INTO core_vulns VALUES (?,?,?,?,?,?)', (ext_vuln[0], ext_vuln[1], ext_vuln[2], ext_vuln[3], ext_vuln[4], ext_vuln[5],))
+                        c.execute('INSERT INTO core_vulns VALUES (?,?,?,?,?,?)', (core_vuln[0], core_vuln[1], core_vuln[2], core_vuln[3], core_vuln[4], core_vuln[5]))
                         conn.commit()
                     else:
                         if update_counter == 0:
                             print('[!] Already up-to-date.\n')
                         else:
-                            print('[+] Done.')
+                            print(' \u2514 Done. Added {} new CORE vulnerabilities to database.\n'.format(update_counter))
-                            print('[!] Added {} new CORE vulnerabilities to database.\n'.format(update_counter))
                         return True
 
     def dlProgress(self, count, blockSize, totalSize):
@@ -191,7 +190,7 @@ class Update:
             infile.close()
             outfile.close()
         except Exception as e:
-            print ('\n', e)
+            print('\n', e)
 
     def load_extensions(self):
         """
@@ -239,13 +238,13 @@ class Update:
         """
         print('\n[+] Searching for new extension vulnerabilities...')
         update_counter = 0
-        response = requests.get('https://typo3.org/help/security-advisories/typo3-extensions/1')
+        response = requests.get('https://typo3.org/help/security-advisories/typo3-extensions/page-1')
-        pages = re.findall('<a class=\"page-link\" href=\"/help/security-advisories/typo3-extensions/([0-9]+)\">', response.text)
+        pages = re.findall('<a class=\"page-link\" href=\"/help/security-advisories/typo3-extensions/page-([0-9]+)\">', response.text)
         last_page = int(pages[-1])
 
         for current_page in range(1, last_page+1):
             print(' \u251c Page {}/{}'.format(current_page, last_page))
-            response = requests.get('https://typo3.org/help/security-advisories/typo3-extensions/{}'.format(current_page), timeout=6)
+            response = requests.get('https://typo3.org/help/security-advisories/typo3-extensions/page-{}'.format(current_page), timeout=6)
             advisories = re.findall('TYPO3-EXT-SA-[0-9][0-9][0-9][0-9]-[0-9][0-9][0-9]', response.text)
             for advisory in advisories:
                 vulnerabilities = []
@@ -275,9 +274,13 @@ class Update:
                         extensionkey = re.findall('Extension[s]?:\s?(.*?)<', beauty_html)
                         # Sometimes there are multiple extensions in an advisory
                         if len(extensionkey) == 0: # If only one extension affected
+                            if not '(' in advisory_info:
+                                extensionkey = [advisory_info[advisory_info.rfind(' ')+1:]]
+                            else:
                                 extensionkey = [advisory_info[advisory_info.find('('):]]
                         for item in range (0, len(extensionkey)):
                             extensionkey_item = extensionkey[item]
+                            if '(' in extensionkey_item:
                                 extensionkey_item = extensionkey_item[extensionkey_item.rfind('(')+1:extensionkey_item.rfind(')')]
                             description = vulnerability[item]
                             version_item = affected_versions[item]

typo3scan.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see [http://www.gnu.org/licenses/](http://www.gnu.org/licenses/)
 #-------------------------------------------------------------------------------
 
-__version__ = '0.6'
+__version__ = '0.6.2'
 __program__ = 'Typo3Scan'
 __description__ = 'Automatic Typo3 enumeration tool'
 __author__ = 'https://github.com/whoot'
@@ -33,18 +33,64 @@ from lib.extensions import Extensions
 from colorama import Fore, init, deinit, Style
 init(strip=False)
 
+from IPython import embed
+
 class Typo3:
     def __init__(self):
         self.__domain_list = []
         self.__path = os.path.dirname(os.path.abspath(__file__))
         self.__extensions = []
 
-    def run(self):
+    def run_magic(self,check,database,conn):
-        if (args.user_agent):
+            # check for typo3 information
-            user_agent = args.user_agent
+            print('\n[+] Typo3 Installation')
+            print('----------------------')
+            check.search_login()
+            check.search_typo3_version()
+
+            # Search extensions
+            print('\n [+] Extension Search')
+            if not self.__extensions:
+                conn = sqlite3.connect(database)
+                c = conn.cursor()
+                if args.vuln:
+                    for row in c.execute('SELECT extensionkey FROM extension_vulns'):
+                        self.__extensions.append(row[0])
+                    self.__extensions = set(self.__extensions)
                 else:
+                    for row in c.execute('SELECT extensionkey FROM extensions'):
+                        self.__extensions.append(row[0])
+                conn.close()
+            print ('  \u251c Brute-Forcing {} Extensions'.format(len(self.__extensions)))
+            extensions = Extensions()
+            ext_list = extensions.search_extension(check.get_path(), self.__extensions, args.threads)
+            if ext_list:
+                #embed()
+                print ('\n  \u251c Found {} extensions'.format(len(ext_list)))
+                print ('  \u251c Brute-Forcing Version Information'.format(len(self.__extensions)))
+                ext_list = extensions.search_ext_version(ext_list, args.threads)
+                json_ext = extensions.output(ext_list, database)
+            else:
+                print ('\n  [!] No extensions found.\n')
+            if args.json:
+                json_log = {}
+                json_log[check.get_name()] = {'Backend': check.get_backend(), 'Version': check.get_typo3_version(), 'Vulnerabilities':check.get_typo3_vulns(), 'Extensions': json_ext}
+                json.dump(json_log, open('typo3scan.json', 'w'))
+
+    def open_database(self):
             database = os.path.join(self.__path, 'lib', 'typo3scan.db')
             conn = sqlite3.connect(database)
+            return database, conn
+
+    def run(self):
+        # use the force!
+        mightyForce = args.d4rkf0rce
+
+        if (args.user_agent):
+            user_agent = args.user_agent
+            database, conn = self.open_database()
+        else:
+            database, conn = self.open_database()
             c = conn.cursor()       
             c.execute('SELECT * FROM UserAgents ORDER BY RANDOM() LIMIT 1;')
             user_agent = c.fetchone()[0]
@@ -71,38 +117,19 @@ class Typo3:
                 default_files = check.check_default_files()
                 if not default_files:
                     check_404 = check.check_404()
-                if not check.is_typo3():
-                    print(Fore.RED + '\n[x] It seems that Typo3 is not used on this domain\n' + Fore.RESET)
-                else:
-                    # check for typo3 information
-                    print('\n[+] Typo3 Installation')
-                    print('----------------------')
-                    check.search_login()
-                    check.search_typo3_version()
 
-                    # Search extensions
+                if not check.is_typo3() and not mightyForce:
-                    print('\n [+] Extension Search')
+                    print(Fore.RED + '\n[x] It seems that Typo3 is not used on this domain\n' + Fore.RESET)
-                    if not self.__extensions:
+                elif not check.is_typo3() and mightyForce:
-                        conn = sqlite3.connect(database)
+                    print(Fore.RED + '\n[x] It seems that Typo3 is not used on this domain' + Fore.RESET)
-                        c = conn.cursor()
-                        if args.vuln:
-                            for row in c.execute('SELECT extensionkey FROM extension_vulns'):
-                                self.__extensions.append(row[0])
-                            self.__extensions = set(self.__extensions)
                 else:
-                            for row in c.execute('SELECT extensionkey FROM extensions'):
+                    self.run_magic(check, database, conn)
-                                self.__extensions.append(row[0])
+
-                        conn.close()
+                if mightyForce==True:
-                    print ('  \u251c Brute-Forcing {} Extensions'.format(len(self.__extensions)))
+                    print(Fore.RED + '[!] I don\'t care and know what i do mode! Yeahhhhhh! Force!!!\n' + Fore.RESET)
-                    extensions = Extensions()
+                    self.run_magic(check,database,conn)
-                    ext_list = extensions.search_extension(check.get_path(), self.__extensions, args.threads)
+
-                    if ext_list:
+                        
-                        print ('\n  \u251c Found {} extensions'.format(len(ext_list)))
-                        print ('  \u251c Brute-Forcing Version Information'.format(len(self.__extensions)))
-                        ext_list = extensions.search_ext_version(ext_list, args.threads)
-                        extensions.output(ext_list, database)
-                    else:
-                        print ('\n  [!] No extensions found.\n')
         except KeyboardInterrupt:
             print('\nReceived keyboard interrupt.\nQuitting...')
             exit(-1)
@@ -155,6 +182,9 @@ Options:
     --threads THREADS   The number of threads to use for enumerating extensions.
                         Default: 5
 
+    --force             I know what i do mode. Test for typo3 anyways.
+    --json              Output results to json file
+
   General:
     -u | --update       Update the database.
     -r | --reset        Reset the database.
@@ -173,6 +203,8 @@ Options:
     parser.add_argument('--cookie', dest='cookie', type=str, default='')
     parser.add_argument('--agent', dest='user_agent', type=str, default='')
     parser.add_argument('--timeout', dest='timeout', type=int, default=10)
+    parser.add_argument('--force', dest='d4rkf0rce', action="store_true", default=False)
+    parser.add_argument('--json', dest='json', action='store_true')
     help.add_argument( '-h', '--help', action='store_true')
     args = parser.parse_args()