Skip to content

Commit

Permalink
Update WebSieve.py
Browse files Browse the repository at this point in the history
Version 1.2
  • Loading branch information
secnextechnologies committed May 9, 2023
1 parent e55fb2c commit b81c9ff
Showing 1 changed file with 35 additions and 29 deletions.
64 changes: 35 additions & 29 deletions WebSieve.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
\/ \/ \/ \/ \/ \/
- SecneX Technologies (secnex.tech)
[INF] Current WebSieve version v1
[INF] Current WebSieve version v1.2
""")

# Set up command line arguments
Expand All @@ -30,36 +30,42 @@
parser.print_help()
exit()

# Convert cookies string to a dictionary
cookies_dict = {}
if args.cookies:
for cookie in args.cookies.split("; "):
key, value = cookie.split("=")
cookies_dict[key] = value
try:
# Convert cookies string to a dictionary
cookies_dict = {}
if args.cookies:
for cookie in args.cookies.split("; "):
key, value = cookie.split("=")
cookies_dict[key] = value

# Convert cookies dictionary to a RequestsCookieJar object
cookies = cookiejar_from_dict(cookies_dict)
# Convert cookies dictionary to a RequestsCookieJar object
cookies = cookiejar_from_dict(cookies_dict)

# Send request and extract URLs
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
response = requests.get(args.url, headers=headers, cookies=cookies)
soup = BeautifulSoup(response.content, "html.parser")
urls = set() # use a set to store unique URLs
# Send request and extract URLs
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
response = requests.get(args.url, headers=headers, cookies=cookies)
response.raise_for_status() # Raise an exception for HTTP errors (e.g., 404)

soup = BeautifulSoup(response.content, "html.parser")
urls = set() # use a set to store unique URLs

for link in soup.find_all('a'):
href = link.get('href')
if href and href.startswith("http"):
urls.add(href)
for link in soup.find_all('a'):
href = link.get('href')
if href and href.startswith("http"):
urls.add(href)

url_list = "\n".join([url for url in urls])
print(url_list)

# Output or save the result
if args.output:
with open(args.output, "w") as f:
f.write(url_list)
print(f"\nURLs saved to {args.output}")
else:
url_list = "\n".join([url for url in urls])
print(url_list)

# Output or save the result
if args.output:
with open(args.output, "w") as f:
f.write(url_list)
print(f"\nURLs saved to {args.output}")
else:
print(url_list)

except requests.exceptions.RequestException as e:
print(f"Error: {str(e)}")

0 comments on commit b81c9ff

Please sign in to comment.