From 7e536efe1b217668585276550db69e5fd79e5368 Mon Sep 17 00:00:00 2001 From: Hitesh <0hitesh.thakur@gmail.com> Date: Sat, 1 Mar 2025 20:15:23 +0530 Subject: [PATCH 1/3] Puredns with custom wordlist --- Dockerfile | 16 +++++++++ configs/local.yml | 2 +- mantis/db/db_models.py | 2 +- mantis/modules/discovery/Puredns.py | 44 +++++++++++++++++++++++++ mantis/tool_base_classes/toolScanner.py | 33 +++++++++++++++++-- 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 mantis/modules/discovery/Puredns.py diff --git a/Dockerfile b/Dockerfile index 098737e8..530ea55b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,22 @@ RUN mv Corsy-1.0-rc Corsy RUN mv Corsy /usr/bin RUN rm -rf * +# Install massdns +RUN echo "Installing massdns" +RUN git clone https://github.com/blechschmidt/massdns.git /massdns +WORKDIR /massdns +RUN make +RUN mv bin/massdns /usr/bin/massdns +WORKDIR / +RUN rm -rf /massdns + +# Install puredns +RUN echo "Installing puredns" +RUN wget https://github.com/d3mondev/puredns/releases/download/v2.1.1/puredns_2.0.1_linux_amd64.tar.gz +RUN tar -xzf puredns_2.1.1_linux_amd64.tar.gz +RUN mv puredns /usr/bin/puredns +RUN rm -rf puredns_2.1.1_linux_amd64.tar.gz + # Install Poetry RUN pip install poetry==1.4.2 diff --git a/configs/local.yml b/configs/local.yml index 83b85dc2..b6b813ca 100644 --- a/configs/local.yml +++ b/configs/local.yml @@ -14,7 +14,7 @@ workflow: cmd: [] workflowConfig: - moduleName : discovery - tools: ['Subfinder','Go_Virustotal','Go_Wayback'] + tools: ['Subfinder','Go_Virustotal','Go_Wayback','Puredns'] order: 1 - moduleName: prerecon tools: ['FindCDN', 'Naabu'] diff --git a/mantis/db/db_models.py b/mantis/db/db_models.py index 08be1c26..8bb7c0b8 100644 --- a/mantis/db/db_models.py +++ b/mantis/db/db_models.py @@ -26,7 +26,7 @@ class Assets(BaseModel): active_hosts: Optional[list] = list() stale: Optional[bool] = False repositories: Optional[str] = Field(None) - tools_source: Optional[str] = Field(None) + tool_source: Optional[str] = Field(None) others: Optional[dict] = dict() diff --git a/mantis/modules/discovery/Puredns.py b/mantis/modules/discovery/Puredns.py new file mode 100644 index 00000000..425c7de9 --- /dev/null +++ b/mantis/modules/discovery/Puredns.py @@ -0,0 +1,44 @@ +from mantis.constants import ASSET_TYPE_SUBDOMAIN +from mantis.utils.crud_utils import CrudUtils +from mantis.tool_base_classes.toolScanner import ToolScanner +from mantis.models.args_model import ArgsModel +from mantis.utils.tool_utils import get_assets_grouped_by_type +from mantis.constants import ASSET_TYPE_TLD + +''' +Subfinder module enumerates subdomain of the TLDs which are fetched from database. +Output file: .txt separated by new line. +Each subdomain discovered is inserted into the database as a new asset. +''' + + +class Puredns(ToolScanner): + + def __init__(self) -> None: + super().__init__() + super().download_required_file() + + + async def get_commands(self, args: ArgsModel): + self.org = args.org + self.base_command = 'puredns bruteforce configs/resources/best-dns-wordlist.txt {input_domain} -r configs/resources/resolvers.txt --write {output_file_path}' + self.outfile_extension = ".txt" + self.assets = await get_assets_grouped_by_type(self, args, ASSET_TYPE_TLD) + return super().base_get_commands(self.assets) + + def parse_report(self, outfile): + output_dict_list = [] + puredns_output = open(outfile).readlines() + for domain in puredns_output: + domain_dict = {} + domain_dict['_id'] = domain.rstrip('\n') + domain_dict['asset'] = domain.rstrip('\n') + domain_dict['asset_type'] = ASSET_TYPE_SUBDOMAIN + domain_dict['org'] = self.org + domain_dict['tool_source'] = "Puredns" + output_dict_list.append(domain_dict) + + return output_dict_list + + async def db_operations(self, tool_output_dict, asset=None): + await CrudUtils.insert_assets(tool_output_dict) \ No newline at end of file diff --git a/mantis/tool_base_classes/toolScanner.py b/mantis/tool_base_classes/toolScanner.py index 15acc242..04fae0ec 100644 --- a/mantis/tool_base_classes/toolScanner.py +++ b/mantis/tool_base_classes/toolScanner.py @@ -1,3 +1,4 @@ +from mantis.utils.base_request import BaseRequestExecutor from mantis.utils.common_utils import CommonUtils from subprocess import Popen, PIPE, DEVNULL from mantis.models.args_model import ArgsModel @@ -5,6 +6,8 @@ import sys import time import asyncio +import os +import requests class ToolScanner: @@ -35,8 +38,34 @@ def base_get_commands(self, assets) : command_list.append((self, command, outfile, every_asset)) self.commands_list = command_list return command_list - - + + def download_required_file(self): + # Define the download directory + download_dir = "config/resources/" + os.makedirs(download_dir, exist_ok=True) + + # Define URLs and corresponding filenames + files = { + "https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt": "resolvers.txt", # https://github.com/trickest + "https://wordlists-cdn.assetnote.io/data/manual/best-dns-wordlist.txt": "best-dns-wordlist.txt" #https://www.assetnote.io/ + } + + for url, filename in files.items(): + file_path = os.path.join(download_dir, filename) + + try: + print(f"Downloading {filename}...") + # _, response = BaseRequestExecutor.sendRequest("GET", url) + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + file.write(chunk) + print(f"Downloaded {filename} successfully.") + except BaseRequestExecutor as e: + print(f"Failed to download {filename}: {e}") + def parse_report(self, outfile): raise NotImplementedError From 8c43e9e261a9af530114c637792a87c69c8e3676 Mon Sep 17 00:00:00 2001 From: Hitesh <0hitesh.thakur@gmail.com> Date: Sat, 1 Mar 2025 20:27:00 +0530 Subject: [PATCH 2/3] modified description for puredns --- mantis/modules/discovery/Puredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mantis/modules/discovery/Puredns.py b/mantis/modules/discovery/Puredns.py index 425c7de9..cd2a7a1a 100644 --- a/mantis/modules/discovery/Puredns.py +++ b/mantis/modules/discovery/Puredns.py @@ -6,7 +6,7 @@ from mantis.constants import ASSET_TYPE_TLD ''' -Subfinder module enumerates subdomain of the TLDs which are fetched from database. +puredns is a fast domain resolver and subdomain bruteforcing tool that can accurately filter out wildcard subdomains and DNS poisoned entries. Output file: .txt separated by new line. Each subdomain discovered is inserted into the database as a new asset. ''' From 2f635311e5f3da11fbe8ffb157a9b8e6449db95c Mon Sep 17 00:00:00 2001 From: Hitesh <0hitesh.thakur@gmail.com> Date: Fri, 7 Mar 2025 14:39:51 +0530 Subject: [PATCH 3/3] added fix for downloading large file in Puredns --- mantis/tool_base_classes/toolScanner.py | 32 ++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/mantis/tool_base_classes/toolScanner.py b/mantis/tool_base_classes/toolScanner.py index 04fae0ec..e5b9339e 100644 --- a/mantis/tool_base_classes/toolScanner.py +++ b/mantis/tool_base_classes/toolScanner.py @@ -41,13 +41,13 @@ def base_get_commands(self, assets) : def download_required_file(self): # Define the download directory - download_dir = "config/resources/" + download_dir = "configs/resources/" os.makedirs(download_dir, exist_ok=True) # Define URLs and corresponding filenames files = { - "https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt": "resolvers.txt", # https://github.com/trickest - "https://wordlists-cdn.assetnote.io/data/manual/best-dns-wordlist.txt": "best-dns-wordlist.txt" #https://www.assetnote.io/ + "https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt": "resolvers.txt", # https://github.com/trickest + "https://wordlists-cdn.assetnote.io/data/manual/best-dns-wordlist.txt": "best-dns-wordlist.txt" # https://www.assetnote.io/ } for url, filename in files.items(): @@ -55,16 +55,32 @@ def download_required_file(self): try: print(f"Downloading {filename}...") - # _, response = BaseRequestExecutor.sendRequest("GET", url) response = requests.get(url, stream=True) - response.raise_for_status() + response.raise_for_status() # Raise an error for bad status codes (4xx, 5xx) + # Get the total file size from the headers + total_size = int(response.headers.get('content-length', 0)) + downloaded_size = 0 + + # Download the file in chunks with open(file_path, "wb") as file: for chunk in response.iter_content(chunk_size=8192): - file.write(chunk) - print(f"Downloaded {filename} successfully.") - except BaseRequestExecutor as e: + if chunk: # Filter out keep-alive chunks + file.write(chunk) + downloaded_size += len(chunk) + # Print download progress + if total_size > 0: + progress = (downloaded_size / total_size) * 100 + print(f"Downloaded {downloaded_size}/{total_size} bytes ({progress:.2f}%)", end="\r") + + print(f"\nDownloaded {filename} successfully.") + + except requests.exceptions.RequestException as e: print(f"Failed to download {filename}: {e}") + # Optionally, delete the partially downloaded file + if os.path.exists(file_path): + os.remove(file_path) + print(f"Deleted partially downloaded file: {filename}") def parse_report(self, outfile): raise NotImplementedError