File tree Expand file tree Collapse file tree 1 file changed +4
-4
lines changed
automaticwebsiteurlscraper.py Expand file tree Collapse file tree 1 file changed +4
-4
lines changed Original file line number Diff line number Diff line change 16
16
page = Request (Url , headers = {'User-Agent' :'Mozilla/5.0' })
17
17
page = urlopen (page , context = ctx , ).read ()
18
18
# Using beautifulsoup to read the contents of the page
19
- soup = BeautifulSoup (page , 'html.parser' )
19
+ soup = BeautifulSoup (page , 'html.parser' )
20
20
# finding all the link headers
21
- links = soup .findAll ('a' )
21
+ links = soup .findAll ('a' )
22
22
if (links is not None ) :
23
- finalLinks = []
23
+ finalLinks = []
24
24
# getting actual site links from the header a
25
25
for link in links :
26
26
if 'href' in str (link ):
27
27
templist = str (link ).split ("href" )
28
28
index1 = templist [- 1 ].index ("\" " )
29
29
index2 = templist [- 1 ][index1 + 1 : ].index ( "\" " )
30
- finalLinks .append (templist [- 1 ][ index1 : index2 + 3 ])
30
+ finalLinks .append (templist [- 1 ][index1 : index2 + 3 ])
31
31
print ("Here are your final links" )
32
32
# printing the final completed list
33
33
for i in finalLinks :
You can’t perform that action at this time.
0 commit comments