#!/usr/bin/python

#Get the source with urllib
from urllib import request
#Parse it with soup
from bs4 import BeautifulSoup
#If you want to use args
import sys
#If you want pager in term
from pydoc import pager

#User agent to use
agent1 = ('Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X)'
        ' AppleWebKit/605.1.15 (KHTML, like Gecko)'
        ' CriOS/108.0.5359.112 Mobile/15E148 Safari/604.1')

agent2 = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0)'
        ' Gecko/20100101 Firefox/109.0')

#Make header for urllib to report
header = {
        'User-Agent': agent1,
        'Accept': 'text/html,application/xhtml+xml,'
        'image/avif,image/webp,image/apng,'
        'application/signed-exchange;v=b3;q=0.9'  
        'application/xml;q=0.9,*/*;q=0.8',
        'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
        'Accept-Language': 'en-US,en;q=0.8',
            #Get a gzip page from server
        #'Accept-Encoding': 'gzip, deflate, br',
        #'Content-type': 'application/json'}
            #urllib will override and send a close
        #'Connection': 'keep-alive'
        }
                
#Simple header
#header = {'User-Agent': agent1}

#Get url of page with input
url = input('Enter/Paste URL :')
#Get url of page with args
#url = str(sys.argv[1])

#Get page, feed it to soup
req = request.Request(url, data=None, headers=header)
html = request.urlopen(req)
soup = BeautifulSoup(html)

#Kill all script and style elements
for script in soup(["script", "style"]):
    script.extract()

#Get text of page
text = soup.get_text()

#Break into lines and remove leading/trailing spaces
lines = (line.strip() for line in text.splitlines())

#Break multi-headlines into a line each
chunks = (phrase.strip() for line in lines for phrase in line.split("  "))

#Drop blank lines, space lines
text = '\n\n'.join(chunk for chunk in chunks if chunk)

#Write file and/or output to term
with open('webpage.txt', 'a') as f:
    f.write(text)

#Pager for scrolling text in terminal with up/down pgUp/pgDown
pager(text)

req = request.Request(url, data=None, headers=header)

req = request.Request(url)

...
urllib.error.HTTPError: HTTP Error 403: Forbidden

#!/usr/bin/python

from urllib import request
import http

user_agent = ('Mozilla/5.0 (Windows NT 10.0; Win64 x64; rv:109.0) '
            'Gecko/20100101 Firefox/109.0')

header = {
        'User-Agent': user_agent,
        'Accept': 'text/html,application/xhtml+xml,'
        'image/avif,image/webp,image/apng,'
        'application/signed-exchange;v=b3;q=0.9'  
        'application/xml;q=0.9,*/*;q=0.8',
        'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
        'Accept-Language': 'en-US,en;q=0.8',
        
        #Get a gzip page from server
        #'Accept-Encoding': 'gzip, deflate, br',
        
        #'Content-type': 'application/json'}
        
        #urllib will override and send a close
        #'Connection': 'keep-alive'
        }

url1 = 'https://www.whatismybrowser.com/detect/what-http-headers-is-my-browser-sending'

req = request.Request(url1, data=None, headers=header)    
    
#5 second timeout for ncat, or any page that won't answer
try:
    resp = request.urlopen(req, timeout=5).read()
    with open('header_test.html', 'wb') as f:
        f.write(resp)

except http.client.HTTPException as e:
    print(e)