webscraping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "webscrapilng.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNl1dvM1kc7NJv/vCNplKcS"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sfvSVNUTM5Lj"
      },
      "outputs": [],
      "source": [
        "import urllib.parse, urllib.error\n",
        "import urllib.request as req\n",
        "from urllib.parse import urlparse\n",
        "from bs4 import BeautifulSoup\n",
        "import requests\n",
        "import re\n",
        "from urllib.request import urlopen,Request\n",
        "from difflib import SequenceMatcher\n",
        "\n",
        "import pandas as pd \n",
        "from pandas import DataFrame\n",
        "import xlrd as xl \n",
        "from pandas import ExcelWriter\n",
        "from pandas import ExcelFile \n",
        "import numpy as np\n",
        "\n",
        "    \n",
        "def open_url(website_link):\n",
        "\ttry:\n",
        "\t\turl = 'https://www.' + website_link + '/'\n",
        "\t\trequest = req.Request(url,headers={'User-Agent' : \"foobar\"})\n",
        "\t\tresponse = req.urlopen(request)\n",
        "\t\tvalid = 1\n",
        "\t\treturn url,valid\n",
        "\texcept:\n",
        "\t\tpass\n",
        "    \n",
        "\ttry:\n",
        "\t\turl = 'http://www.' + website_link + '/'\n",
        "\t\trequest = req.Request(url,headers={'User-Agent' : \"foobar\"})\n",
        "\t\tresponse = req.urlopen(request)\n",
        "\t\tvalid = 1\n",
        "\t\treturn url,valid\n",
        "\texcept:\n",
        "\t\turl = 'https://www.' + website_link + '/'\n",
        "\t\tvalid = 0\n",
        "\t\tresponse = 'website does not exist'\n",
        "\t\treturn url,valid  \n",
        "\n",
        "def validity_check(u_check):\n",
        "    try:\n",
        "        request = req.Request(u_check,headers = {'User-Agent' : \"foobar\"})\n",
        "        response = req.urlopen(request)\n",
        "        return 1\n",
        "    except:\n",
        "        return 0 \n",
        "\n",
        "\n",
        "def similar(a,b):\n",
        "    return SequenceMatcher(None,a,b).ratio()\n",
        "\n",
        "bad_urls = [\"NULL\", \"_blank\", \"None\", None, \"NoneType\"]\n",
        "bad_chars = [';',':','!',\"*\",\"=\",'{','}','[',']',\"©\",'@','^','_','-',',','+','(',')',\"&\",\"%\",\"#\",'/',\" \",\"\t\",\"    \",'\\n','\\t',\"\\\\\",'\"','|',\"'\",\"$\",\"·\",\"–\",\"<\",\">\",\"?\"]\n",
        "\n",
        "#path1 = 'C:\\Users\\anous\\OneDrive\\Desktop\\input.xls' need forward slashes\n",
        "\n",
        "sheet1 = pd.read_excel(r\"input.xlsx\", sheet_name = \"Sheet1\")\n",
        "\n",
        "cycle_count = 0\n",
        "for ind,row in sheet1.iterrows():\n",
        "    \n",
        "    cycle_count += 1\n",
        "    print(cycle_count,ind)\n",
        "\n",
        "    website_name = row['website'] #domain_name -- column name\n",
        "\n",
        "    url,valid = open_url(website_name)\n",
        "\n",
        "    print(url,valid)\n",
        "\n",
        "    print('Valid',valid)\n",
        "    sheet1.loc[ind,'Valid'] = valid\n",
        "\n",
        "\n",
        "    if(valid == 1):\n",
        "        try:\n",
        "            r1 = requests.get(url)\n",
        "\n",
        "            final_url = r1.url\n",
        "\n",
        "            print('Final Url', final_url)\n",
        "            sheet1.loc[ind,'Final Url'] = final_url\n",
        "\n",
        "            redirected = (url!=r1.url)\n",
        "\n",
        "            similarity_index = similar(url,final_url)  #how similar address 1 and final address are #remove https and www from the urls before matching\n",
        "\n",
        "           # print('Similarity Index', similarity_index)\n",
        "            sheet1.loc[ind,'Similarity Index'] = similarity_index\n",
        "\n",
        "            if(redirected == True):\n",
        "                print('Redirected = 1')\n",
        "                sheet1.loc[ind,'Redirected'] = 1\n",
        "                url = final_url\n",
        "            else:\n",
        "       #         print('Redirected = 0')\n",
        "                sheet1.loc[ind,'Redirected'] = 0\n",
        "\n",
        "            if 'domain' in url or 'godaddy' in url: #list of websites that sell or register domains\n",
        "      #          print('For sale = 1')\n",
        "                sheet1.loc[ind,'For sale'] = 1\n",
        "            else:\n",
        "          #      print('For sale = 0')\n",
        "                sheet1.loc[ind,'For sale'] = 0\n",
        "\n",
        "            req_page = req.Request(url,headers={'User-Agent' : \"foobar\"})\n",
        "            html = req.urlopen(req_page).read()\n",
        "            soup = BeautifulSoup(html,'html.parser') #searching for tags through this\n",
        "            \n",
        "            lang1 = soup.find('html') #<html lang='en-US'>\n",
        "            #print(lang1)\n",
        "\n",
        "\n",
        "            language = lang1.get('lang')\n",
        "            print(language)\n",
        "\n",
        "            sheet1.loc[ind,'Language'] = language\n",
        "\n",
        "\n",
        "            links = []\n",
        "\n",
        "            base_url = url\n",
        "\n",
        "            for link1 in soup.find_all('a'): #<a href='/contact-us'> <a href=''> \n",
        "                link = link1.get('href')\n",
        "                if link == 'None':\n",
        "                    continue\n",
        "            \n",
        "                if link not in links and link not in bad_urls:\n",
        "\n",
        "                    if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                        pass\n",
        "                    else:\n",
        "                        try:\n",
        "                            if link[0] == \"/\":\n",
        "                                link=link[1:]\n",
        "                        except Exception as e:\n",
        "                            print(e,'101 line error')\n",
        "                        if base_url in link:\n",
        "                            if base_url == link:\n",
        "                                pass\n",
        "                            if base_url != link and \"https://\"in link:\n",
        "                                link=link[len(base_url)-1:]\n",
        "                        try:\n",
        "                            if link[0] == \"/\":\n",
        "                                link=link[1:]\n",
        "                        except Exception as e:\n",
        "                            print(e,'101 line error')\n",
        "\n",
        "                        if \"http://\" in link:\n",
        "                            pass\n",
        "                        elif \"https://\" in link:\n",
        "                            pass\n",
        "                        else:\n",
        "                            link = base_url + link\n",
        "\n",
        "                #print(link)\n",
        "                if link not in links:\n",
        "                    links.append(link)\n",
        "\n",
        "            url_p = urlparse(url)\n",
        "\n",
        "            pages = [] #internal links\n",
        "            external_links = []\n",
        "\n",
        "\n",
        "            for link in links:\n",
        "                try:\n",
        "                    p_link = urlparse(link)\n",
        "\n",
        "                    if url_p.netloc in link:\n",
        "                        if link not in pages:\n",
        "                            if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                                links.remove(link)\n",
        "                            else:\n",
        "                                pages.append(link)\n",
        "                    else:\n",
        "                        if link not in external_links:\n",
        "                            external_links.append(link)\n",
        "                except:\n",
        "                    links.remove(link)\n",
        "\n",
        "\n",
        "            pages_1 = []\n",
        "\n",
        "            links_2 = []\n",
        "\n",
        "            for page in pages:\n",
        "\n",
        "                v = validity_check(page)\n",
        "\n",
        "                if(v==1):\n",
        "                    try:\n",
        "                        r = req.Request(page,headers={'User-Agent' : \"foobar\"})\n",
        "                        r_open = req.urlopen(r).read()\n",
        "                        soup1 = BeautifulSoup(r_open,'html.parser')\n",
        "\n",
        "                        for link1 in soup1.find_all('a'):\n",
        "                            link = link1.get('href')\n",
        "                            if link not in links and link not in bad_urls:\n",
        "                                if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                                    pass\n",
        "                                else:\n",
        "                                    try:\n",
        "                                        if link[0] == \"/\":\n",
        "                                            link=link[1:]\n",
        "                                    except Exception as e:\n",
        "                                        print(e,'160 line error')\n",
        "                                    if base_url in link:\n",
        "                                        if base_url == link:\n",
        "                                            pass\n",
        "                                        if base_url != link and \"https://\"in link:\n",
        "                                            link=link[len(base_url)-1:]\n",
        "                                    try:\n",
        "                                        if link[0] == \"/\":\n",
        "                                            link=link[1:]\n",
        "                                    except Exception as e:\n",
        "                                        print(e,'170 line error')\n",
        "\n",
        "                                    if link not in bad_urls:\n",
        "                                        if \"http://\" in link:\n",
        "                                            pass\n",
        "                                        elif \"https://\" in link:\n",
        "                                            pass\n",
        "                                        else:\n",
        "                                            link = base_url + link\n",
        "\n",
        "                            #print(link)\n",
        "                            if link not in links and link not in bad_urls:\n",
        "                                if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                                    pass\n",
        "                                else:                            \n",
        "                                    links.append(link)\n",
        "                                    links_2.append(link)\n",
        "\n",
        "                    except: \n",
        "                        pass\n",
        "\n",
        "            if base_url not in links:  \n",
        "                len_url = len(base_url) - 2\n",
        "                base_url2 = base_url[0:len_url]\n",
        "                if base_url2 not in links:\n",
        "                    links.append(base_url)\n",
        "                    pages.append(base_url)\n",
        "\n",
        "\n",
        "            for link in links_2:\n",
        "                if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                    links_2.remove(link)\n",
        "                else:\n",
        "                    print('235')\n",
        "                    try:\n",
        "                        p_link = urlparse(link)\n",
        "\n",
        "                        if url_p.netloc in link:\n",
        "                            if link not in pages:\n",
        "                                if \".jpg\" in link or \".png\" in link or \".gif\" in link or \".jpeg\" in link:\n",
        "                                    pass\n",
        "                                else:\n",
        "                                    pages.append(link)\n",
        "                                    pages_1.append(link)\n",
        "                        else:\n",
        "                            if link not in external_links:\n",
        "                                external_links.append(link)\n",
        "                    except:\n",
        "                        links.remove(link)\n",
        "                        links_2.remove(link)\n",
        "\n",
        "\n",
        "         #   print('254')\n",
        "            for page in pages_1:\n",
        "                v = validity_check(page)\n",
        "        #        print('257')\n",
        "                if(v==1):\n",
        "                    try:\n",
        "                        r = req.Request(page,headers={'User-Agent' : \"foobar\"})\n",
        "                        r_open = req.urlopen(r).read()\n",
        "                        soup1 = BeautifulSoup(r_open,'html.parser')\n",
        "\n",
        "                    except:\n",
        "                        pass\n",
        "\n",
        "\n",
        "            number_total_links = len(links)\n",
        "            print('Total Links', number_total_links)   \n",
        "\n",
        "            print('pages')\n",
        "            print(pages)\n",
        "            print('external links')\n",
        "            print(external_links)\n",
        "            number_pages = len(pages)\n",
        "            number_links = len(external_links)\n",
        "\n",
        "            print('Internal Pages', number_pages)\n",
        "            print('External Links', number_links)\n",
        "            sheet1.loc[ind,'Internal Pages'] = number_pages\n",
        "            sheet1.loc[ind,'External Links'] = number_links\n",
        "\n",
        "\n",
        "            count_invalid_external = 0\n",
        "\n",
        "            for link in external_links: #add more elements here if you want #find a way to see if these are related to the domain or not\n",
        "\n",
        "                if link not in bad_urls:                     \n",
        "                    v = validity_check(link)\n",
        "                    if(v==0):\n",
        "                        count_invalid_external += 1\n",
        "\n",
        "            print('Invalid External Links',count_invalid_external)\n",
        "\n",
        "            sheet1.loc[ind,'Invalid External Links'] = count_invalid_external\n",
        "\n",
        "\n",
        "            words = 0\n",
        "            count_invalid_internal = 0\n",
        "\n",
        "\n",
        "            for link in pages:\n",
        "\n",
        "                if link not in bad_urls:\n",
        "\n",
        "                    v = validity_check(link)\n",
        "\n",
        "                    if(v==1):\n",
        "                        try:\n",
        "                            #print('here')\n",
        "                            r = req.Request(link,headers={'User-Agent' : \"foobar\"})\n",
        "                            html = req.urlopen(r).read()\n",
        "                            soup1 = BeautifulSoup(html, features=\"html.parser\")\n",
        "\n",
        "                            text = soup1.get_text()\n",
        "\n",
        "                            lines = (line.strip() for line in text.splitlines())\n",
        "                            chunks = (phrase.strip() for line in lines for phrase in line.split(\"  \"))\n",
        "                            text = '\\n'.join(chunk for chunk in chunks if chunk)\n",
        "                            text_list = text.split()\n",
        "\n",
        "\n",
        "                            text_list2 = []\n",
        "                            count_false = 0\n",
        "\n",
        "                            for word in text_list:\n",
        "                                if '@' in word:\n",
        "                                    pass\n",
        "                                else:\n",
        "                                    for i in bad_chars:\n",
        "                                        word = word.replace(i,'')\n",
        "                                        word = word.replace('\\\\','')\n",
        "                                        #print(word)\n",
        "                                    if word.strip() == '':\n",
        "                                        pass\n",
        "                                    elif word.strip() == '-':\n",
        "                                        pass\n",
        "                                    elif word.strip() == '\"':\n",
        "                                        pass\n",
        "                                    else:\n",
        "                                        if(len(word)>0):\n",
        "                                            text_list2.append(word)\n",
        "\n",
        "                            words += len(text_list2)\n",
        "\n",
        "                        except Exception as e:\n",
        "                            v = 0\n",
        "                            print(e,'internal page')\n",
        "\n",
        "                    if(v==0):\n",
        "                        count_invalid_internal += 1 \n",
        "\n",
        "                else:\n",
        "                    pass\n",
        "\n",
        "\n",
        "            #keywords search #seo analysis\n",
        "\n",
        "            print('Invalid Internal Links', count_invalid_internal)\n",
        "            print('Total Words', words)\n",
        "            #print('Spelling Errors', count_errors)\n",
        "            sheet1.loc[ind,'Invalid Internal Links'] = count_invalid_internal\n",
        "            sheet1.loc[ind,'Total Words'] = words\n",
        "           # string_pages= ', '.join([str(elem) for elem in pages])\n",
        "           # string_ext= ', '.join([str(elem) for elem in external_links])\n",
        "           # sheet1.loc[ind,'Internal Links'] = string_pages\n",
        "           # sheet1.loc[ind,'External Links'] = string_ext\n",
        "            #sheet1.loc[ind,'Spelling Errors'] = count_errors\n",
        "            textfile = open(\"all_links.txt\", \"w\")\n",
        "            for element in pages:\n",
        "                textfile.write(element + \"\\n\")\n",
        "            for elem in external_links:\n",
        "                textfile.write(element + \"\\n\")\n",
        "            textfile.close()\n",
        "\n",
        "        except Exception as e:\n",
        "            print(e,'complete exception')\n",
        "            valid = 0\n",
        "\n",
        "    if (valid == 0):\n",
        "        print('valid', valid)\n",
        "        sheet1.loc[ind,'Valid'] = 0\n",
        "        sheet1.loc[ind,'Language'] = 0\n",
        "        sheet1.loc[ind,'Redirected'] = 0\n",
        "        sheet1.loc[ind,'Final Url'] = url\n",
        "        sheet1.loc[ind,'Similarity Index'] = 1\n",
        "        sheet1.loc[ind,'For sale'] = 0\n",
        "        sheet1.loc[ind,'Total Links'] = 0\n",
        "        sheet1.loc[ind,'Internal Pages'] = 0\n",
        "        sheet1.loc[ind,'External Links'] = 0\n",
        "        sheet1.loc[ind,'Invalid External Links'] = 0\n",
        "        sheet1.loc[ind,'Invalid Internal Links'] = 0\n",
        "        sheet1.loc[ind,'Total Words'] = 0\n",
        "        \n",
        "\n",
        "    sheet1.to_excel(r\"output.xls\", sheet_name='Scraped Data') #path will have forward slashes"
      ]
    }
  ]
}