Merge pull request #2888 from blacklanternsecurity/dev

Dev -> Stable 2.8.1

Author: TheTechromancer

bbot/core/event/base.py

@@ -1247,6 +1247,10 @@ def _words(self):
         return set()
 
 
+class OPEN_UDP_PORT(OPEN_TCP_PORT):
+    pass
+
+
 class URL_UNVERIFIED(BaseEvent):
     _status_code_regex = re.compile(r"^status-(\d{1,3})$")
 

bbot/modules/azure_tenant.py

@@ -1,6 +1,3 @@
-import regex as re
-from contextlib import suppress
-
 from bbot.modules.base import BaseModule
 
 
@@ -9,116 +6,90 @@ class azure_tenant(BaseModule):
     produced_events = ["DNS_NAME"]
     flags = ["affiliates", "subdomain-enum", "cloud-enum", "passive", "safe"]
     meta = {
-        "description": "Query Azure for tenant sister domains",
+        "description": "Query Azure via azmap.dev for tenant sister domains",
         "created_date": "2024-07-04",
         "author": "@TheTechromancer",
     }
 
-    base_url = "https://autodiscover-s.outlook.com"
+    base_url = "https://azmap.dev/api/tenant"
     in_scope_only = True
     per_domain_only = True
 
     async def setup(self):
         self.processed = set()
-        self.d_xml_regex = re.compile(r"<Domain>([^<>/]*)</Domain>", re.I)
         return True
 
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
-        domains, openid_config = await self.query(query)
-
-        tenant_id = None
-        authorization_endpoint = openid_config.get("authorization_endpoint", "")
-        matches = await self.helpers.re.findall(self.helpers.regexes.uuid_regex, authorization_endpoint)
-        if matches:
-            tenant_id = matches[0]
-
-        tenant_names = set()
-        if domains:
-            self.verbose(f'Found {len(domains):,} domains under tenant for "{query}": {", ".join(sorted(domains))}')
-            for domain in domains:
+        tenant_data = await self.query(query)
+
+        if not tenant_data:
+            return
+
+        tenant_id = tenant_data.get("tenant_id")
+        tenant_name = tenant_data.get("tenant_name")
+        email_domains = tenant_data.get("email_domains", [])
+
+        if email_domains:
+            self.verbose(
+                f'Found {len(email_domains):,} domains under tenant for "{query}": {", ".join(sorted(email_domains))}'
+            )
+            for domain in email_domains:
                 if domain != query:
                     await self.emit_event(
                         domain,
                         "DNS_NAME",
                         parent=event,
                         tags=["affiliate", "azure-tenant"],
-                        context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {{event.data}}',
+                        context=f'{{module}} queried azmap.dev for "{query}" and found {{event.type}}: {{event.data}}',
                     )
-                    # tenant names
-                    if domain.lower().endswith(".onmicrosoft.com"):
-                        tenantname = domain.split(".")[0].lower()
-                        if tenantname:
-                            tenant_names.add(tenantname)
-
-            tenant_names = sorted(tenant_names)
-            event_data = {"tenant-names": tenant_names, "domains": sorted(domains)}
+
+            # Build tenant names list (include the tenant name from the API)
+            tenant_names = []
+            if tenant_name:
+                tenant_names.append(tenant_name)
+
+            # Also extract tenant names from .onmicrosoft.com domains
+            for domain in email_domains:
+                if domain.lower().endswith(".onmicrosoft.com"):
+                    tenantname = domain.split(".")[0].lower()
+                    if tenantname and tenantname not in tenant_names:
+                        tenant_names.append(tenantname)
+
+            event_data = {"tenant-names": tenant_names, "domains": sorted(email_domains)}
             tenant_names_str = ",".join(tenant_names)
-            if tenant_id is not None:
+            if tenant_id:
                 event_data["tenant-id"] = tenant_id
             await self.emit_event(
                 event_data,
                 "AZURE_TENANT",
                 parent=event,
-                context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {tenant_names_str}',
+                context=f'{{module}} queried azmap.dev for "{query}" and found {{event.type}}: {tenant_names_str}',
             )
 
     async def query(self, domain):
-        url = f"{self.base_url}/autodiscover/autodiscover.svc"
-        data = f"""<?xml version="1.0" encoding="utf-8"?>
-<soap:Envelope xmlns:exm="http://schemas.microsoft.com/exchange/services/2006/messages" xmlns:ext="http://schemas.microsoft.com/exchange/services/2006/types" xmlns:a="http://www.w3.org/2005/08/addressing" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
-    <soap:Header>
-        <a:Action soap:mustUnderstand="1">http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetFederationInformation</a:Action>
-        <a:To soap:mustUnderstand="1">https://autodiscover-s.outlook.com/autodiscover/autodiscover.svc</a:To>
-        <a:ReplyTo>
-            <a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>
-        </a:ReplyTo>
-    </soap:Header>
-    <soap:Body>
-        <GetFederationInformationRequestMessage xmlns="http://schemas.microsoft.com/exchange/2010/Autodiscover">
-            <Request>
-                <Domain>{domain}</Domain>
-            </Request>
-        </GetFederationInformationRequestMessage>
-    </soap:Body>
-</soap:Envelope>"""
-
-        headers = {
-            "Content-Type": "text/xml; charset=utf-8",
-            "SOAPAction": '"http://schemas.microsoft.com/exchange/2010/Autodiscover/Autodiscover/GetFederationInformation"',
-            "User-Agent": "AutodiscoverClient",
-            "Accept-Encoding": "identity",
-        }
+        url = f"{self.base_url}?domain={domain}&extract=true"
 
         self.debug(f"Retrieving tenant domains at {url}")
 
-        autodiscover_task = self.helpers.create_task(
-            self.helpers.request(url, method="POST", headers=headers, content=data)
-        )
-        openid_url = f"https://login.windows.net/{domain}/.well-known/openid-configuration"
-        openid_task = self.helpers.create_task(self.helpers.request(openid_url))
-
-        r = await autodiscover_task
+        r = await self.helpers.request(url)
         status_code = getattr(r, "status_code", 0)
-        if status_code not in (200, 421):
+        if status_code != 200:
             self.verbose(f'Error retrieving azure_tenant domains for "{domain}" (status code: {status_code})')
-            return set(), {}
-        found_domains = list(set(await self.helpers.re.findall(self.d_xml_regex, r.text)))
-        domains = set()
+            return {}
 
-        for d in found_domains:
-            # make sure we don't make any unnecessary api calls
+        try:
+            tenant_data = r.json()
+        except Exception as e:
+            self.warning(f'Error parsing JSON response for "{domain}": {e}')
+            return {}
+
+        # Absorb domains into word cloud
+        email_domains = tenant_data.get("email_domains", [])
+        for d in email_domains:
             d = str(d).lower()
             _, query = self.helpers.split_domain(d)
             self.processed.add(hash(query))
-            domains.add(d)
-            # absorb into word cloud
             self.scan.word_cloud.absorb_word(d)
 
-        r = await openid_task
-        openid_config = {}
-        with suppress(Exception):
-            openid_config = r.json()
-
-        domains = sorted(domains)
-        return domains, openid_config
+        return tenant_data

bbot/modules/censys_dns.py

@@ -0,0 +1,82 @@
+from bbot.modules.templates.censys import censys
+
+
+class censys_dns(censys):
+    """
+    Query the Censys certificates API for subdomains.
+    Thanks to https://github.com/owasp-amass/amass/blob/master/resources/scripts/cert/censys.ads
+    """
+
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query the Censys API for subdomains",
+        "created_date": "2022-08-04",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "max_pages": 5}
+    options_desc = {
+        "api_key": "Censys.io API Key in the format of 'key:secret'",
+        "max_pages": "Maximum number of pages to fetch (100 results per page)",
+    }
+
+    async def setup(self):
+        self.max_pages = self.config.get("max_pages", 5)
+        return await super().setup()
+
+    async def query(self, query):
+        results = set()
+        cursor = ""
+        for i in range(self.max_pages):
+            url = f"{self.base_url}/v2/certificates/search"
+            json_data = {
+                "q": f"names: {query}",
+                "per_page": 100,
+            }
+            if cursor:
+                json_data.update({"cursor": cursor})
+            resp = await self.api_request(
+                url,
+                method="POST",
+                json=json_data,
+            )
+
+            if resp is None:
+                break
+
+            try:
+                d = resp.json()
+            except Exception as e:
+                self.warning(f"Failed to parse JSON from {url} (response: {resp}): {e}")
+
+            if resp.status_code < 200 or resp.status_code >= 400:
+                if isinstance(d, dict):
+                    error = d.get("error", "")
+                    if error:
+                        self.warning(error)
+                self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i + 1}')
+                self.debug(f"Response: {resp.text}")
+                break
+            else:
+                if d is None:
+                    break
+                elif not isinstance(d, dict):
+                    break
+                status = d.get("status", "").lower()
+                result = d.get("result", {})
+                hits = result.get("hits", [])
+                if status != "ok" or not hits:
+                    break
+
+                for h in hits:
+                    names = h.get("names", [])
+                    for n in names:
+                        results.add(n.strip(".*").lower())
+
+                cursor = result.get("links", {}).get("next", "")
+                if not cursor:
+                    break
+
+        return results