HTTP Error 403: Forbidden

[exportio]

Traceback (most recent call last):
File "main.py", line 21, in
links = crawler.start()
File "\crawler.py", line 17, in start
self.crawl(self.url)
File "\crawler.py", line 26, in crawl
response = urllib.request.urlopen(url)
File "\Python36_64\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "\Python36_64\lib\urllib\request.py", line 532, in open
response = meth(req, response)
File "\Python36_64\lib\urllib\request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "\Python36_64\lib\urllib\request.py", line 570, in error
return self._call_chain(*args)
File "\Python36_64\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "\Python36_64\lib\urllib\request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden

[Aminsaffar]

This is probably because of mod_security or some similar server security feature which blocks known spider/bot user agents
change this line of code

		req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
		response = urlopen(req)
		#response = urllib.request.urlopen(url)
		page = str(response.read())