Skip to content

Commit 1358790

Browse files
OSINT-TECHNOLOGIESOSINT-TECHNOLOGIES
authored
Stabilized v1.3.2
2 parents 1833baa + 02f4d55 commit 1358790

File tree

7 files changed

+492
-201
lines changed

7 files changed

+492
-201
lines changed

‎datagather_modules/data_assembler.py‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def report_preprocessing(self, short_domain, report_file_type):
6666
def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, keywords, keywords_flag, dorking_flag, used_api_flag, snapshotting_flag, username, from_date, end_date):
6767
casename, db_casename, db_creation_date, robots_filepath, sitemap_filepath, sitemap_links_filepath, report_file_type, report_folder, ctime, report_ctime = self.report_preprocessing(short_domain, report_file_type)
6868
logging.info(f'### THIS LOG PART FOR {casename} CASE, TIME: {ctime} STARTS HERE')
69-
print(Fore.GREEN + "Started scanning domain" + Style.RESET_ALL)
69+
print(Fore.LIGHTMAGENTA_EX + "\n[STARTED BASIC DOMAIN SCAN]" + Style.RESET_ALL)
7070
print(Fore.GREEN + "[1/11] Getting domain IP address" + Style.RESET_ALL)
7171
ip = cp.ip_gather(short_domain)
7272
print(Fore.GREEN + '[2/11] Gathering WHOIS information' + Style.RESET_ALL)
@@ -111,16 +111,15 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
111111
total_ports = len(ports)
112112
total_ips = len(subdomain_ip) + 1
113113
total_vulns = len(vulns)
114-
115-
print(Fore.LIGHTMAGENTA_EX + "\n[BASIC SCAN END]\n" + Style.RESET_ALL)
114+
print(Fore.LIGHTMAGENTA_EX + "[ENDED BASIC DOMAIN SCAN]\n" + Style.RESET_ALL)
116115
if report_file_type == 'xlsx':
117116
if pagesearch_flag.lower() == 'y':
118117
if subdomains[0] != 'No subdomains were found':
119118
to_search_array = [subdomains, social_medias, sd_socials]
120-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
119+
print(Fore.LIGHTMAGENTA_EX + "[STARTED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
121120
ps_emails_return, accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, website_elements_counter, exposed_passwords_counter, keywords_messages_list = subdomains_parser(to_search_array[0], report_folder, keywords, keywords_flag)
122121
total_links_counter = accessed_links_counter = "No results because PageSearch does not gather these categories"
123-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
122+
print(Fore.LIGHTMAGENTA_EX + "[ENDED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
124123
else:
125124
print(Fore.RED + "Cant start PageSearch because no subdomains were detected")
126125
accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = total_links_counter = accessed_links_counter = emails_amount = 'No results because no subdomains were found'
@@ -136,13 +135,13 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
136135
dorking_results = ['Google Dorking mode was not selected for this scan']
137136
else:
138137
dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
139-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
138+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
140139
dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, get_dorking_query(short_domain, dorking_db_path, table))
141-
print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
140+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
142141

143142
api_scan_db = []
144143
if used_api_flag != ['Empty']:
145-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
144+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
146145
if '1' in used_api_flag:
147146
virustotal_output = api_virustotal_check(short_domain)
148147
api_scan_db.append('VirusTotal')
@@ -163,7 +162,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
163162
securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
164163
if '3' not in used_api_flag:
165164
hudsonrock_output = 'No results because user did not selected HudsonRock API scan'
166-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
165+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
167166
else:
168167
virustotal_output = 'No results because user did not selected VirusTotal API scan'
169168
securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
@@ -173,14 +172,14 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
173172
if snapshotting_flag.lower() in ['s', 'p', 'w']:
174173
config_values = read_config()
175174
installed_browser = config_values['installed_browser']
176-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
175+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
177176
if snapshotting_flag.lower() == 's':
178177
take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
179178
elif snapshotting_flag.lower() == 'p':
180179
save_page_as_html(url, report_folder + '//domain_html_copy.html')
181180
elif snapshotting_flag.lower() == 'w':
182181
download_snapshot(short_domain, from_date, end_date, report_folder)
183-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
182+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
184183
else:
185184
pass
186185

@@ -199,7 +198,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
199198
if pagesearch_flag.lower() == 'y':
200199
if subdomains[0] != 'No subdomains were found':
201200
to_search_array = [subdomains, social_medias, sd_socials]
202-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
201+
print(Fore.LIGHTMAGENTA_EX + "[STARTED EXTENDED DOMAIN SCAN WITH PAGESEARCH]" + Style.RESET_ALL)
203202
(
204203
ps_emails_return,
205204
accessible_subdomains,
@@ -214,7 +213,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
214213
total_links_counter = accessed_links_counter = "No results because PageSearch does not gather these categories"
215214
if len(keywords_messages_list) == 0:
216215
keywords_messages_list = ['No keywords were found']
217-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
216+
print(Fore.LIGHTMAGENTA_EX + "[ENDED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
218217
else:
219218
print(Fore.RED + "Cant start PageSearch because no subdomains were detected")
220219
ps_emails_return = ""
@@ -233,13 +232,13 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
233232
dorking_file_path = 'Google Dorking mode was not selected for this scan'
234233
else:
235234
dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
236-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
235+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]" + Style.RESET_ALL)
237236
dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, get_dorking_query(short_domain, dorking_db_path, table))
238-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
237+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
239238

240239
api_scan_db = []
241240
if used_api_flag != ['Empty']:
242-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
241+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]" + Style.RESET_ALL)
243242
if '1' in used_api_flag:
244243
virustotal_output = api_virustotal_check(short_domain)
245244
api_scan_db.append('VirusTotal')
@@ -260,7 +259,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
260259
securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
261260
if '3' not in used_api_flag:
262261
hudsonrock_output = 'No results because user did not selected HudsonRock API scan'
263-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
262+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
264263
else:
265264
virustotal_output = 'No results because user did not selected VirusTotal API scan'
266265
securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
@@ -270,14 +269,14 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
270269
if snapshotting_flag.lower() in ['s', 'p', 'w']:
271270
config_values = read_config()
272271
installed_browser = config_values['installed_browser']
273-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
272+
print(Fore.LIGHTMAGENTA_EX + f"[STARTED DOMAIN SNAPSHOTTING]" + Style.RESET_ALL)
274273
if snapshotting_flag.lower() == 's':
275274
take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
276275
elif snapshotting_flag.lower() == 'p':
277276
save_page_as_html(url, report_folder + '//domain_html_copy.html')
278277
elif snapshotting_flag.lower() == 'w':
279278
download_snapshot(short_domain, from_date, end_date, report_folder)
280-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
279+
print(Fore.LIGHTMAGENTA_EX + f"[ENDED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
281280
else:
282281
pass
283282

‎dpulse.py‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import sys
22
import os
33
from colorama import Fore, Style, Back
4+
from rich.progress import Progress, SpinnerColumn, TextColumn
45

56
sys.path.append('datagather_modules')
67
sys.path.append('service')
@@ -75,24 +76,29 @@ def process_report(report_filetype, short_domain, url, case_comment, keywords_li
7576
spinner_thread.do_run = False
7677
spinner_thread.join()
7778

78-
class ProgressBar(threading.Thread):
79+
80+
class RichProgressBar(threading.Thread):
7981
def __init__(self):
80-
super(ProgressBar, self).__init__()
82+
super(RichProgressBar, self).__init__()
8183
self.do_run = True
8284

8385
def run(self):
84-
for char in itertools.cycle('|/-\\'):
85-
if not self.do_run:
86-
break
87-
print(Fore.LIGHTMAGENTA_EX + Back.WHITE + char + Style.RESET_ALL, end='\r')
88-
sleep(0.1)
86+
with Progress(
87+
SpinnerColumn(),
88+
TextColumn("[magenta]Processing scan...[/magenta]"),
89+
transient=True,
90+
) as progress:
91+
task = progress.add_task("", total=None)
92+
while self.do_run:
93+
progress.update(task)
94+
sleep(0.1)
8995

9096
def run():
9197
while True:
9298
try:
9399
cli.print_main_menu()
94100
domain_patter = r'^(?!\-)(?:[a-zA-Z0-9\-]{1,63}\.)+[a-zA-Z]{2,}$'
95-
choice = input(Fore.YELLOW + "Enter your choice >> ")
101+
choice = input(Fore.YELLOW + "\nEnter your choice >> ")
96102
if choice == "1":
97103
from misc import domain_precheck
98104
print(Fore.GREEN + "\nImported and activated reporting modules" + Style.RESET_ALL)
@@ -208,8 +214,8 @@ def run():
208214
end_date = str(input('Enter end date (YYYYMMDD format): '))
209215
snapshotting_ui_mark = "Yes, domain's main page snapshotting using Wayback Machine"
210216
cli_init.print_prescan_summary(short_domain, report_filetype.upper(), pagesearch_ui_mark, dorking_ui_mark, used_api_ui, case_comment, snapshotting_ui_mark)
211-
print(Fore.LIGHTMAGENTA_EX + "[BASIC SCAN START]\n" + Style.RESET_ALL)
212-
spinner_thread = ProgressBar()
217+
#print(Fore.LIGHTMAGENTA_EX + "[BASIC SCAN START]\n" + Style.RESET_ALL)
218+
spinner_thread = RichProgressBar()
213219
spinner_thread.start()
214220
if report_filetype.lower() in ['html', 'xlsx']:
215221
process_report(report_filetype, short_domain, url, case_comment,
@@ -221,7 +227,7 @@ def run():
221227
elif choice == "2":
222228
import configparser
223229
cli.print_settings_menu()
224-
choice_settings = input(Fore.YELLOW + "Enter your choice >> ")
230+
choice_settings = input(Fore.YELLOW + "\nEnter your choice >> ")
225231
if choice_settings == '1':
226232
print_and_return_config()
227233
elif choice_settings == '2':
@@ -249,7 +255,7 @@ def run():
249255
continue
250256
elif choice == '3':
251257
cli.dorking_db_manager()
252-
choice_dorking = input(Fore.YELLOW + "Enter your choice >> ")
258+
choice_dorking = input(Fore.YELLOW + "\nEnter your choice >> ")
253259
if choice_dorking == '1':
254260
from db_creator import manage_dorks
255261
cli_init.print_api_db_msg()

‎pagesearch/pagesearch_parsers.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def clean_bad_pdfs(ps_docs_path):
6565
def subdomains_parser(subdomains_list, report_folder, keywords, keywords_flag):
6666
report_lines: List[str] = []
6767
p = make_recorder(report_lines)
68-
print(Fore.GREEN + "Conducting PageSearch. Please, be patient, it may take a long time\n" + Style.RESET_ALL)
68+
#print(Fore.GREEN + "Conducting PageSearch. Please, be patient, it may take a long time\n" + Style.RESET_ALL)
6969
ps_docs_path = os.path.join(report_folder, 'ps_documents')
7070
if not os.path.exists(ps_docs_path):
7171
os.makedirs(ps_docs_path)
@@ -195,6 +195,7 @@ def subdomains_parser(subdomains_list, report_folder, keywords, keywords_flag):
195195
keywords_messages_list = ['No keywords were found because of None user input']
196196
else:
197197
print(Fore.GREEN + f"[+] Total {pdf_with_keywords} keywords were found in PDF files" + Style.RESET_ALL)
198+
p(Fore.LIGHTGREEN_EX + "-------------------------------------------------" + Style.RESET_ALL)
198199

199200
data_tuple = (
200201
ps_emails_return,

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "dpulse"
7-
version = "1.3.1"
7+
version = "1.3.2"
88
description = "Convenient,fast and user-friendly collector of domain information from Open-Sources"
99
authors = ["OSINT-TECHNOLOGIES <osint.technologies@gmail.com>"]
1010
readme = "README.md"

‎reporting_modules/html_report_creation.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ def report_assembling(short_domain, url, case_comment, data_array, report_info_a
8888
virustotal_output = virustotal_html_prep(virustotal_output)
8989
securitytrails_output = securitytrails_html_prep(securitytrails_output)
9090

91-
9291
if len(ps_emails_return) > 0:
9392
subdomain_mails += ps_emails_return
9493
subdomain_mails = list(set(subdomain_mails))

0 commit comments

Comments
 (0)