Skip to content

Commit 782c39d

Browse files
TheTechromancerTheTechromancer
authored
Merge pull request #2884 from blacklanternsecurity/reenable-censys
Re-enable Censys
2 parents 38bda97 + b7c0215 commit 782c39d

8 files changed

Lines changed: 698 additions & 2 deletions

File tree

‎bbot/core/event/base.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,6 +1247,10 @@ def _words(self):
12471247
return set()
12481248

12491249

1250+
class OPEN_UDP_PORT(OPEN_TCP_PORT):
1251+
pass
1252+
1253+
12501254
class URL_UNVERIFIED(BaseEvent):
12511255
_status_code_regex = re.compile(r"^status-(\d{1,3})$")
12521256

‎bbot/modules/censys_dns.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
from bbot.modules.templates.censys import censys
2+
3+
4+
class censys_dns(censys):
5+
"""
6+
Query the Censys certificates API for subdomains.
7+
Thanks to https://github.com/owasp-amass/amass/blob/master/resources/scripts/cert/censys.ads
8+
"""
9+
10+
watched_events = ["DNS_NAME"]
11+
produced_events = ["DNS_NAME"]
12+
flags = ["subdomain-enum", "passive", "safe"]
13+
meta = {
14+
"description": "Query the Censys API for subdomains",
15+
"created_date": "2022-08-04",
16+
"author": "@TheTechromancer",
17+
"auth_required": True,
18+
}
19+
options = {"api_key": "", "max_pages": 5}
20+
options_desc = {
21+
"api_key": "Censys.io API Key in the format of 'key:secret'",
22+
"max_pages": "Maximum number of pages to fetch (100 results per page)",
23+
}
24+
25+
async def setup(self):
26+
self.max_pages = self.config.get("max_pages", 5)
27+
return await super().setup()
28+
29+
async def query(self, query):
30+
results = set()
31+
cursor = ""
32+
for i in range(self.max_pages):
33+
url = f"{self.base_url}/v2/certificates/search"
34+
json_data = {
35+
"q": f"names: {query}",
36+
"per_page": 100,
37+
}
38+
if cursor:
39+
json_data.update({"cursor": cursor})
40+
resp = await self.api_request(
41+
url,
42+
method="POST",
43+
json=json_data,
44+
)
45+
46+
if resp is None:
47+
break
48+
49+
try:
50+
d = resp.json()
51+
except Exception as e:
52+
self.warning(f"Failed to parse JSON from {url} (response: {resp}): {e}")
53+
54+
if resp.status_code < 200 or resp.status_code >= 400:
55+
if isinstance(d, dict):
56+
error = d.get("error", "")
57+
if error:
58+
self.warning(error)
59+
self.verbose(f'Non-200 Status code: {resp.status_code} for query "{query}", page #{i + 1}')
60+
self.debug(f"Response: {resp.text}")
61+
break
62+
else:
63+
if d is None:
64+
break
65+
elif not isinstance(d, dict):
66+
break
67+
status = d.get("status", "").lower()
68+
result = d.get("result", {})
69+
hits = result.get("hits", [])
70+
if status != "ok" or not hits:
71+
break
72+
73+
for h in hits:
74+
names = h.get("names", [])
75+
for n in names:
76+
results.add(n.strip(".*").lower())
77+
78+
cursor = result.get("links", {}).get("next", "")
79+
if not cursor:
80+
break
81+
82+
return results

‎bbot/modules/censys_ip.py‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
from bbot.modules.templates.censys import censys
2+
3+
4+
class censys_ip(censys):
5+
"""
6+
Query the Censys /v2/hosts/{ip} endpoint for associated hostnames, IPs, and URLs.
7+
"""
8+
9+
watched_events = ["IP_ADDRESS"]
10+
produced_events = [
11+
"IP_ADDRESS",
12+
"DNS_NAME",
13+
"URL_UNVERIFIED",
14+
"OPEN_TCP_PORT",
15+
"OPEN_UDP_PORT",
16+
"TECHNOLOGY",
17+
"PROTOCOL",
18+
]
19+
flags = ["passive", "safe"]
20+
meta = {
21+
"description": "Query the Censys API for hosts by IP address",
22+
"created_date": "2026-01-26",
23+
"author": "@TheTechromancer",
24+
"auth_required": True,
25+
}
26+
options = {"api_key": "", "dns_names_limit": 100, "in_scope_only": True}
27+
options_desc = {
28+
"api_key": "Censys.io API Key in the format of 'key:secret'",
29+
"dns_names_limit": "Maximum number of DNS names to extract from dns.names (default 100)",
30+
"in_scope_only": "Only query in-scope IPs. If False, will query up to distance 1.",
31+
}
32+
scope_distance_modifier = 1
33+
34+
async def setup(self):
35+
self.dns_names_limit = self.config.get("dns_names_limit", 100)
36+
self.warning(
37+
"This module may consume a lot of API queries. Unless you specifically want to query on each individual IP, we recommend using the censys_dns module instead."
38+
)
39+
return await super().setup()
40+
41+
async def filter_event(self, event):
42+
in_scope_only = self.config.get("in_scope_only", True)
43+
max_scope_distance = 0 if in_scope_only else (self.scan.scope_search_distance + 1)
44+
if event.scope_distance > max_scope_distance:
45+
return False, "event is not in scope"
46+
return True
47+
48+
async def handle_event(self, event):
49+
ip = str(event.host)
50+
url = f"{self.base_url}/v2/hosts/{ip}"
51+
52+
resp = await self.api_request(url)
53+
if resp is None:
54+
self.debug(f"No response for {ip}")
55+
return
56+
57+
if resp.status_code == 404:
58+
self.debug(f"No data found for {ip}")
59+
return
60+
61+
if resp.status_code != 200:
62+
self.verbose(f"Non-200 status code ({resp.status_code}) for {ip}")
63+
return
64+
65+
try:
66+
data = resp.json()
67+
except Exception as e:
68+
self.warning(f"Failed to parse JSON response for {ip}: {e}")
69+
return
70+
71+
result = data.get("result", {})
72+
if not result:
73+
return
74+
75+
# Track what we've already emitted to avoid duplicates
76+
seen = set()
77+
78+
# Extract data from services
79+
for service in result.get("services", []):
80+
port = service.get("port")
81+
transport = service.get("transport_protocol", "TCP").upper()
82+
83+
# Emit OPEN_TCP_PORT or OPEN_UDP_PORT for services with a port
84+
# QUIC uses UDP as transport, so treat it as UDP
85+
if port and (port, transport) not in seen:
86+
seen.add((port, transport))
87+
if transport in ("UDP", "QUIC"):
88+
event_type = "OPEN_UDP_PORT"
89+
else:
90+
event_type = "OPEN_TCP_PORT"
91+
await self.emit_event(
92+
self.helpers.make_netloc(ip, port),
93+
event_type,
94+
parent=event,
95+
context="{module} found open port on {event.parent.data}",
96+
)
97+
98+
# Emit PROTOCOL for non-HTTP services
99+
# Use extended_service_name (more specific) falling back to service_name
100+
# Also check transport_protocol for protocols like QUIC
101+
service_name = service.get("extended_service_name") or service.get("service_name", "")
102+
# If service_name is UNKNOWN but transport_protocol is meaningful, use that
103+
if service_name.upper() == "UNKNOWN" and transport and transport not in ("TCP", "UDP"):
104+
service_name = transport
105+
if service_name and service_name.upper() not in ("HTTP", "HTTPS", "UNKNOWN"):
106+
protocol_key = ("protocol", service_name.upper(), port)
107+
if protocol_key not in seen:
108+
seen.add(protocol_key)
109+
protocol_data = {"host": str(event.host), "protocol": service_name}
110+
if port:
111+
protocol_data["port"] = port
112+
await self.emit_event(
113+
protocol_data,
114+
"PROTOCOL",
115+
parent=event,
116+
context="{module} found {event.type}: {event.data[protocol]} on {event.parent.data}",
117+
)
118+
119+
# Extract URLs from HTTP services
120+
http_data = service.get("http", {})
121+
request = http_data.get("request", {})
122+
uri = request.get("uri")
123+
if uri and uri not in seen:
124+
seen.add(uri)
125+
await self.emit_event(
126+
uri,
127+
"URL_UNVERIFIED",
128+
parent=event,
129+
context="{module} found {event.data} in HTTP service of {event.parent.data}",
130+
)
131+
132+
# Extract TLS certificate data
133+
tls_data = service.get("tls", {})
134+
certs = tls_data.get("certificates", {})
135+
leaf_data = certs.get("leaf_data", {})
136+
137+
# Extract names from leaf_data.names
138+
for name in leaf_data.get("names", []):
139+
await self._emit_host(name, event, seen, "TLS certificate")
140+
141+
# Extract common_name from leaf_data.subject
142+
subject = leaf_data.get("subject", {})
143+
for cn in subject.get("common_name", []):
144+
await self._emit_host(cn, event, seen, "TLS certificate subject")
145+
146+
# Extract software/technologies
147+
for software in service.get("software", []):
148+
product = software.get("uniform_resource_identifier", software.get("product", ""))
149+
if product:
150+
await self.emit_event(
151+
{"technology": product, "host": str(event.host)},
152+
"TECHNOLOGY",
153+
parent=event,
154+
context="{module} found {event.type}: {event.data[technology]} on {event.parent.data}",
155+
)
156+
157+
# Extract dns.names (limit to configured max)
158+
dns_data = result.get("dns", {})
159+
dns_names = dns_data.get("names", [])
160+
for name in dns_names[: self.dns_names_limit]:
161+
await self._emit_host(name, event, seen, "reverse DNS")
162+
163+
async def _emit_host(self, host, event, seen, source):
164+
"""Emit IP_ADDRESS or DNS_NAME for a host value."""
165+
# Validate and emit as DNS_NAME
166+
try:
167+
validated = self.helpers.validators.validate_host(host)
168+
except ValueError as e:
169+
self.debug(f"Error validating host {host} in {source}: {e}")
170+
if validated and validated not in seen:
171+
seen.add(validated)
172+
await self.emit_event(
173+
validated,
174+
"DNS_NAME",
175+
parent=event,
176+
context=f"{{module}} found {{event.data}} in {source} of {{event.parent.data}}",
177+
)

‎bbot/modules/templates/censys.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import traceback
2+
3+
from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
4+
5+
6+
class censys(subdomain_enum_apikey):
7+
"""
8+
Base template for Censys API modules.
9+
Provides common authentication and API request handling.
10+
"""
11+
12+
options = {"api_key": ""}
13+
options_desc = {"api_key": "Censys.io API Key in the format of 'key:secret'"}
14+
15+
base_url = "https://search.censys.io/api"
16+
17+
async def setup(self):
18+
await super().setup()
19+
api_keys = set()
20+
for module_name in ("censys", "censys_dns", "censys_ip"):
21+
module_config = self.scan.config.get("modules", {}).get(module_name, {})
22+
api_key = module_config.get("api_key", "")
23+
if isinstance(api_key, str):
24+
api_key = [api_key]
25+
for key in api_key:
26+
key = key.strip()
27+
if key:
28+
api_keys.add(key)
29+
if not api_keys:
30+
if self.auth_required:
31+
return None, "No API key set"
32+
self.api_key = api_keys.pop() if api_keys else ""
33+
try:
34+
await self.ping()
35+
self.hugesuccess("API is ready")
36+
return True
37+
except Exception as e:
38+
self.trace(traceback.format_exc())
39+
return None, f"Error with API ({str(e).strip()})"
40+
41+
async def ping(self):
42+
url = f"{self.base_url}/v1/account"
43+
resp = await self.api_request(url, retry_on_http_429=False)
44+
d = resp.json()
45+
assert isinstance(d, dict), f"Invalid response from {url}: {resp}"
46+
quota = d.get("quota", {})
47+
used = int(quota.get("used", 0))
48+
allowance = int(quota.get("allowance", 0))
49+
assert used < allowance, "No quota remaining"
50+
51+
def prepare_api_request(self, url, kwargs):
52+
api_id, api_secret = self.api_key.split(":", 1)
53+
kwargs["auth"] = (api_id, api_secret)
54+
return url, kwargs

‎bbot/modules/wpscan.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@ class wpscan(BaseModule):
5858
},
5959
{
6060
"name": "Install wpscan gem",
61-
# we install globally because installing to a user's home dir is unpredictable across different distros and often missing from PATH
62-
"shell": "gem install wpscan --no-user-install",
61+
"gem": {"name": "wpscan", "state": "latest", "user_install": False},
6362
"become": True,
6463
},
6564
]

‎bbot/test/test_step_1/test_helpers.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
409409
assert helpers.validators.validate_host("LOCALHOST ") == "localhost"
410410
assert helpers.validators.validate_host(" 192.168.1.1") == "192.168.1.1"
411411
assert helpers.validators.validate_host(" Dead::c0dE ") == "dead::c0de"
412+
assert helpers.validators.validate_host(".*.wildcard.evilcorp.com") == "wildcard.evilcorp.com"
412413
assert helpers.validators.soft_validate(" evilCorp.COM", "host") is True
413414
assert helpers.validators.soft_validate("!@#$", "host") is False
414415
with pytest.raises(ValueError):

0 commit comments

Comments
 (0)