-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbasic-scraper.py
58 lines (52 loc) · 23.7 KB
/
basic-scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
from bs4 import BeautifulSoup
import urllib2
import json
import re
import time
def match_class(target):
def do_match(tag):
classes = tag.get('class', [])
return all(c in classes for c in target)
return do_match
##
##Takes a search term and searches all of the craigslist cities and dumps
##a JSON containing the location-based data and results
##
##
queryTerm=('node', 'angular', 'mongodb', 'd3','postgress')
for term in queryTerm:
searchList= ["kenai","juneau","humboldt","mendocino","siskiyou","danville","clovis","ventura","orangecounty","imperial","mohave","boulder","denver","lasalle","pueblo","swks","nd","galveston","monroe","boone","tuscarawas","ashtabula","chautauqua","pennstate","greatfalls","mansfield","oneonta","seattle","reno","gulfport","tricities","akroncanton","spacecoast","elmira","nh","monterey","minneapolis","panamacity","gainesville","ocala","lakeland","jacksonville","orlando","daytona","duluth","lakecity","westky","fairbanks","roseburg","eugene","salem","portland","bellingham","redding","chico","klamath","yubasutter","sacramento","bend","stockton","modesto","slo","susanville","yakima","merced","santamaria","wenatchee","santabarbara","visalia","moseslake","bakersfield","losangeles","sanmarcos","palmsprings","meridian","boise","elko","lasvegas","yuma","kalispell","missoula","stgeorge","prescott","phoenix","helena","saltlakecity","logan","bozeman","tucson","sierravista","showlow","billings","farmington","lascruces","albuquerque","elpaso","cosprings","lubbock","amarillo","bismarck","northplatte","sanangelo","abilene","laredo","fredericksburg","wichitafalls","sanantonio","lawton","grandisland","austin","oklahomacity","brownsville","corpuschristi","wichita","waco","grandforks","stillwater","victoriatx","lincoln","ksu","siouxcity","tulsa","topeka","lawrence","bemidji","stjoseph","kansascity","joplin","fortsmith","fayar","brainerd","fortdodge","stcloud","texarkana","mankato","shreveport","ames","desmoines","springfield","lakecharles","masoncity","kirksville","rmn","littlerock","lafayette","cedarrapids","iowacity","eauclaire","lacrosse","batonrouge","houma","jonesboro","dubuque","stlouis","jxn","neworleans","memphis","springfieldil","wausau","peoria","madison","hattiesburg","rockford","janesville","decatur","jacksontn","greenbay","milwaukee","sheboygan","chicago","terrehaute","clarksville","pensacola","owensboro","tippecanoe","nashville","bloomington","muskegon","indianapolis","kokomo","holland","louisville","grandrapids","kalamazoo","cookeville","dothan","chattanooga","battlecreek","fortwayne","columbusga","richmondin","lansing","cincinnati","lexington","atlanta","annarbor","flint","bgky","toledo","monroemi","athensga","valdosta","columbus","chillicothe","sandusky","asheville","athensohio","augusta","statesboro","cleveland","charlestonwv","columbia","charlotte","youngstown","winstonsalem","meadville","erie","pittsburgh","morgantown","roanoke","charleston","greensboro","florencesc","lynchburg","myrtlebeach","fayetteville","buffalo","harrisonburg","charlottesville","winchester","wilmington","rochester","richmond","onslow","frederick","washingtondc","williamsport","harrisburg","york","baltimore","auburn","ithaca","annapolis","lancaster","syracuse","reading","binghamton","philadelphia","newyork","albany","glensfalls","boston","savannah","northernwi","norfolk","odessa","cenla","centralmich","evansville","knoxville","eastky","sarasota","hickory","poconos","smd","montana","tampa","wheeling","honolulu","keys","hiltonhead","westmd","outerbanks","newjersey","allentown","burlington","nwga","olympic","twinfalls","butte","eastoregon","quadcities","sandiego","corvallis","southjersey","chambana","westslope","loz","inlandempire","goldcountry","hartford","eastidaho","eastco","enid","sd","dallas","texoma","nwks","beaumont","semo","shoals","tuscaloosa","bham","okaloosa","wyoming","nmi","tallahassee","macon","cfl","treasure","montgomery","newhaven","cnj","jerseyshore","nwct","fresno","spokane","jackson","natchez","carbondale","gadsden","detroit","martinsburg","newlondon","limaohio","mobile","fargo","bn","eastnc","miami","pullman","waterloo","santafe","collegestation","flagstaff","provo","huntsville","houston","oregoncoast","easttexas","swv","easternshore","fingerlakes","delaware","anchorage","medford","skagit","sfbay","hanford","kpr","lewiston","ogden","rockies","fortcollins","roswell","bigbend","scottsbluff","rapidcity","delrio","csd","mcallen","killeen","nesd","siouxfalls","omaha","marshall","seks","nacogdoches","columbiamo","ottumwa","quincy","northmiss","appleton","mattoon","racine","up","southbend","swmi","muncie","thumb","albanyga","dayton","saginaw","huntington","porthuron","greenville","swva","fortmyers","zanesville","parkersburg","staugustine","wv","blacksburg","raleigh","altoona","chambersburg","twintiers","scranton","utica","potsdam","catskills","hudsonvalley","plattsburgh","longisland","worcester","providence","watertown","southcoast","westernmass","capecod","maine","salina","brunswick"]
cityList= ["Kenai Peninsula, AK","Southeast Alaska, AK","Humboldt County, CA","Mendocino County, CA","Siskiyou County, CA","Danville, CA","Clovis / Portales, CA","Ventura County, CA","Orange County, CA","Imperial County, CA","Mohave County, AZ","Boulder, CO","Denver, CO","La Salle Co, CO","Pueblo, CO","Southwest Ks, KS","North Dakota, ND","Galveston, TX","Monroe, LA","Boone, NC","Tuscarawas Co, OH","Ashtabula, OH","Chautauqua, NY","State College, PA","Great Falls, VA","Mansfield, PA","Oneonta, NY","Seattle-Tacoma, WA","Reno / Tahoe, NV","Gulfport / Biloxi, MS","Tri-Cities, TN","Akron / Canton, OH","Space Coast, FL","Elmira-Corning, NY","New Hampshire, NH","Monterey Bay, CA","Minneapolis / St Paul, MN","Panama City, FL","Gainesville, FL","Ocala, FL","Lakeland, FL","Jacksonville, FL","Orlando, FL","Daytona Beach, FL","Duluth / Superior, MN","North Central Fl, FL","Western Ky, KY","Fairbanks, AK","Roseburg, OR","Eugene, OR","Salem, OR","Portland, OR","Bellingham, WA","Redding, CA","Chico, CA","Klamath Falls, OR","Yuba-Sutter, CA","Sacramento, CA","Bend, OR","Stockton, CA","Modesto, CA","San Luis Obispo, CA","Susanville, CA","Yakima, WA","Merced, CA","Santa Maria, CA","Wenatchee, WA","Santa Barbara, CA","Visalia-Tulare, CA","Moses Lake, WA","Bakersfield, CA","Los Angeles, CA","San Marcos, CA","Palm Springs, CA","Meridian, ID","Boise, ID","Elko, NV","Las Vegas, NV","Yuma, AZ","Kalispell, MT","Missoula, MT","St George, UT","Prescott, AZ","Phoenix, AZ","Helena, MT","Salt Lake City, UT","Logan, UT","Bozeman, MT","Tucson, AZ","Sierra Vista, AZ","Show Low, AZ","Billings, MT","Farmington, NM","Las Cruces, NM","Albuquerque, NM","El Paso, TX","Colorado Springs, CO","Lubbock, TX","Amarillo, TX","Bismarck, ND","North Platte, NE","San Angelo, TX","Abilene, TX","Laredo, TX","Fredericksburg, TX","Wichita Falls, TX","San Antonio, TX","Lawton, OK","Grand Island, NE","Austin, TX","Oklahoma City, OK","Brownsville, TX","Corpus Christi, TX","Wichita, KS","Waco, TX","Grand Forks, ND","Stillwater, OK","Victoria, TX","Lincoln, NE","Manhattan, KS","Sioux City, IA","Tulsa, OK","Topeka, KS","Lawrence, KS","Bemidji, MN","St Joseph, MI","Kansas City, MO","Joplin, MO","Fort Smith, AR","Fayetteville, AR","Brainerd, MN","Fort Dodge, IA","St Cloud, MN","Texarkana, TX","Mankato, MN","Shreveport, LA","Ames, IA","Des Moines, IA","Springfield, MO","Lake Charles, LA","Mason City, IA","Kirksville, MO","Rochester, MN","Little Rock, AR","Lafayette, LA","Cedar Rapids, MI","Iowa City, IA","Eau Claire, WI","La Crosse, WI","Baton Rouge, LA","Houma, LA","Jonesboro, AR","Dubuque, IA","St Louis, MO","Jackson, MI","New Orleans, LA","Memphis, TN","Springfield, IL","Wausau, WI","Peoria, IL","Madison, WI","Hattiesburg, MI","Rockford, IL","Janesville, WI","Decatur, IL","Jackson, TN","Green Bay, WI","Milwaukee, WI","Sheboygan, WI","Chicago, IL","Terre Haute, IN","Clarksville, TN","Pensacola, FL","Owensboro, KY","Lafayette / West Lafayette, IN","Nashville, TN","Bloomington, IN","Muskegon, MI","Indianapolis, IN","Kokomo, IN","Holland, MI","Louisville, KY","Grand Rapids, MI","Kalamazoo, MI","Cookeville, TN","Dothan, AL","Chattanooga, TN","Battle Creek, MI","Fort Wayne, IN","Columbus, GA","Richmond, IN","Lansing, MI","Cincinnati, OH","Lexington, KY","Atlanta, GA","Ann Arbor, MI","Flint, MI","Bowling Green, OH","Toledo, OH","Monroe, MI","Athens, GA","Valdosta, GA","Columbus, OH","Chillicothe, OH","Sandusky, OH","Asheville, NC","Athens, OH","Augusta, GA","Statesboro, GA","Cleveland, OH","Charleston, WV","Columbia, SC","Charlotte, NC","Youngstown, OH","Winston-Salem, NC","Meadville, PA","Erie, PA","Pittsburgh, PA","Morgantown, WV","Roanoke, VA","Charleston, SC","Greensboro, NC","Florence, SC","Lynchburg, VA","Myrtle Beach, SC","Fayetteville, NC","Buffalo, NY","Harrisonburg, VA","Charlottesville, VA","Winchester, VA","Wilmington, NC","Rochester, NY","Richmond, VA","Jacksonville, NC","Frederick, MD","Washington, DC","Williamsport, PA","Harrisburg, PA","York, PA","Baltimore, MD","Auburn, NY","Ithaca, NY","Annapolis, IN","Lancaster, PA","Syracuse, NY","Reading, PA","Binghamton, NY","Philadelphia, PA","New York City, NY","Albany, NY","Glens Falls, NY","Boston, MA","Savannah / Hinesville, GA","Northern Wi, WI","Hampton Roads, VA","Odessa / Midland, TX","Central Louisiana, LA","Central Michigan, MI","Evansville, IN","Knoxville, KY","Eastern Kentucky, KY","Sarasota-Bradenton, FL","Hickory / Lenoir, NC","Poconos, PA","Southern Maryland, MD","Eastern Montana, MT","Tampa Bay Area, FL","Northern Panhandle, IL","Hawaii, HI","Florida Keys, FL","Hilton Head, SC","Western Maryland, MD","Outer Banks, NC","North Jersey, NJ","Lehigh Valley, PA","Vermont, VT","Northwest Ga, GA","Olympic Peninsula, WA","Twin Falls, MN","Butte, OR","East Oregon, OR","Quad Cities, IA/IL","San Diego, CA","Corvallis/Albany, OR","South Jersey, NJ","Champaign Urbana, IL","Western Slope, CO","Lake Of The Ozarks, MO","Inland Empire, CA","Gold Country, CA","Hartford, CT","East Idaho, ID","Eastern Co, CO","Northwest Ok, OK","South Dakota, SD","Dallas / Fort Worth, TX","Texoma, TX","Northwest Ks, KS","Beaumont / Port Arthur, TX","Southeast Missouri, MO","Florence / Muscle Shoals, AL","Tuscaloosa, AL","Birmingham, AL","Okaloosa / Walton, FL","Wyoming, WY","Northern Michigan, MI","Tallahassee, TN","Macon / Warner Robins, GA","Heartland Florida, FL","Treasure Coast, FL","Montgomery, AL","New Haven, CT","Central Nj, NJ","Jersey Shore, NJ","Northwest Ct, CT","Fresno / Madera, CA","Spokane / Coeur D'Alene, WA","Jackson, MS","Southwest Ms, MS","Southern Illinois, IL","Gadsden-Anniston, AL","Detroit Metro, MI","Eastern Panhandle, WV","Eastern Ct, CT","Lima / Findlay, OH","Mobile, AL","Fargo / Moorhead, ND","Bloomington-Normal, IL","Eastern Nc, NC","South Florida, FL","Pullman / Moscow, WA","Waterloo / Cedar Falls, IA","Santa Fe / Taos, NM","College Station, TX","Flagstaff / Sedona, AZ","Provo / Orem, UT","Huntsville / Decatur, AL","Houston, TX","Oregon Coast, OR","Tyler / East Tx, TX","Southern Wv, WV","Eastern Shore, MD","Finger Lakes, NY","Delaware, DE","Anchorage / Mat-Su, AK","Medford-Ashland, OR","Skagit / Island / Sji, WA","Sf Bay Area, CA","Hanford-Corcoran, CA","Kennewick-Pasco-Richland, WA","Lewiston / Clarkston, ID","Ogden-Clearfield, UT","High Rockies, CO","Fort Collins / North Co, CO","Roswell / Carlsbad, NM","Southwest Tx, TX","Scottsbluff / Panhandle, NE","Rapid City / West Sd, SD","Del Rio / Eagle Pass, TX","Pierre / Central Sd, SD","Mcallen / Edinburg, TX","Killeen / Temple / Ft Hood, TX","Northeast Sd, SD","Sioux Falls / Se Sd, SD","Omaha / Council Bluffs, NE","Southwest Mn, MN","Southeast Ks, KS","Deep East Texas, TX","Columbia / Jeff City, MO","Southeast Ia, IA","Western Il, IL","North Mississippi, MI","Appleton-Oshkosh-Fdl, WI","Mattoon-Charleston, IL","Kenosha-Racine, WI","Upper Peninsula, MI","South Bend / Michiana, IN","Southwest Michigan, MI","Muncie / Anderson, IN","The Thumb, MI","Albany, GA","Dayton / Springfield, OH","Saginaw-Midland-Baycity, MI","Huntington-Ashland, WV","Port Huron, MI","Greenville / Upstate, SC","Southwest Va, VA","Ft Myers / Sw Florida, FL","Zanesville / Cambridge, MA","Parkersburg-Marietta, WV","St Augustine, FL","West Virginia Old, WV","New River Valley, VA","Raleigh / Durham / Ch, NC","Altoona-Johnstown, PA","Cumberland Valley, PA","Twin Tiers Ny/Pa, NY","Scranton / Wilkes-Barre, PA","Utica-Rome-Oneida, NY","Potsdam-Canton-Massena, NY","Catskills, NY","Hudson Valley, NY","Plattsburgh-Adirondacks, NY","Long Island, NY","Worcester / Central Ma, MA","Rhode Island, RI","Watertown, MA","South Coast, MA","Western Massachusetts, MA","Cape Cod / Islands, MA","Maine, ME","Salina, KS","Brunswick, GA"]
resultSet=[]
locationSet=[(60.0968272,-151.788033),(64.4459613,-149.680909),(40.5997415,-123.8997734),(39.3176491,-123.4126399),(41.6472886,-122.5225194),(37.8215929,-121.9999606),(36.830451,-119.700776),(34.4458248,-119.0779359),(33.7213739,-117.7903245),(33.0305487,-115.3595666),(35.7218038,-113.8271174),(40.102555,-105.3632079),(39.7348381,-104.9653271),(40.350566,-104.7038625),(38.187635,-104.5350145),(37.1850331,-100.8928434),(47.6201461,-100.540737),(29.299328,-94.7945882),(38.2722313,-90.1792484),(36.216795,-81.6745517),(40.3759469,-81.464888),(41.7167229,-80.74947),(42.2894671,-79.421728),(40.794026,-77.8606975),(38.9981653,-77.2883157),(41.805687,-77.0792881),(42.46732115,-75.01716368),(47.4475673,-122.3080658),(39.4987727,-119.7737299),(30.40711375,-89.07143292),(36.47345205,-82.40844247),(40.91522255,-81.43991569),(28.5142066,-80.7990149),(42.1608441,-76.89203499),(43.7782479,-71.7536197),(36.7999557,-121.9010668),(44.92311225,-93.05638403),(30.165156,-85.6605595),(29.67360855,-82.32341406),(29.1924213,-82.1352047),(28.0470698,-81.9539369),(30.327715,-81.6631332),(28.5479786,-81.41278409),(29.209288,-81.0226483),(46.835457,-92.017926),(30.212655,-82.775813),(36.971323,-88.366506),(64.8377802,-147.7163952),(43.216505,-123.3417381),(44.0505054,-123.0950506),(44.9391565,-123.033121),(45.5202471,-122.6741949),(48.754402,-122.4788602),(40.5863563,-122.3916754),(39.7284945,-121.8374777),(42.224867,-121.7816704),(39.1312985,-121.6267423),(38.5815719,-121.4943996),(44.0581728,-121.3153096),(37.9577016,-121.2907796),(37.6390972,-120.9968782),(35.2827525,-120.6596156),(40.4162842,-120.6530063),(46.596449,-120.5531796),(37.3029568,-120.4843269),(34.9143525,-120.4095176),(47.4234599,-120.3103494),(34.4004935,-119.7029107),(36.3302284,-119.2920585),(47.1301417,-119.2780771),(35.3738712,-119.0194639),(34.0536781,-118.2427025),(33.1433723,-117.1661449),(33.8345281,-116.5389475),(43.6086295,-116.392326),(43.615046,-116.2044429),(40.8324212,-115.7631233),(36.1662859,-115.149225),(32.665167,-114.4759049),(48.2022563,-114.3167117),(46.8700801,-113.9952796),(37.104153,-113.5841313),(34.5400242,-112.4685025),(33.4467681,-112.0756724),(46.5927122,-112.036109),(40.7670126,-111.8904308),(41.7313447,-111.8348631),(45.6799842,-111.0446748),(32.2217422,-110.9264759),(31.5545401,-110.3036929),(34.2404365,-110.0422196),(45.7874957,-108.49607),(36.7304288,-108.2089191),(32.3140354,-106.7798078),(35.0841034,-106.6509851),(31.7600372,-106.487287),(38.8339578,-104.8253485),(33.6725165,-101.8580142),(35.2072185,-101.8338246),(46.8083268,-100.7837392),(41.1238873,-100.7654232),(31.4319925,-100.4545797),(32.4464534,-99.7333478),(27.5060991,-99.5088979),(30.2686767,-98.87253269),(33.915899,-98.5067297),(29.4246002,-98.4951405),(34.6086854,-98.3903305),(40.9250124,-98.342007),(30.2711286,-97.7436995),(35.4729886,-97.5170536),(25.9140256,-97.4890856),(27.8002542,-97.3955744),(37.6922361,-97.3375448),(31.549333,-97.1466695),(47.9078244,-97.0592028),(36.1156306,-97.0585717),(28.8297135,-96.97253692),(40.8000554,-96.6674005),(39.1836082,-96.5716694),(42.4966815,-96.4058782),(36.1524357,-95.9904093),(39.0490111,-95.6775557),(38.9719384,-95.2359496),(47.4786541,-94.890802),(39.7686055,-94.8466322),(39.0844687,-94.5630298),(37.0842271,-94.513281),(35.3872218,-94.4248983),(36.111508,-94.20986672),(46.3580221,-94.2008288),(42.4974694,-94.1680158),(45.5616075,-94.1642004),(33.4417695,-94.07382038),(44.1634663,-93.9993505),(32.5221828,-93.7651944),(42.0346917,-93.6201507),(41.5910641,-93.6037149),(37.2153307,-93.2982521),(30.2265949,-93.2173759),(43.1535728,-93.2010367),(40.1947539,-92.5832496),(44.0226213,-92.4630094),(34.7464809,-92.2895948),(30.2240897,-92.0198427),(41.9758872,-91.6704053),(41.6612561,-91.5299106),(44.811349,-91.4984941),(43.8013556,-91.2395807),(30.4507462,-91.154551),(29.5957696,-90.7195348),(35.81457925,-90.66822465),(42.5006217,-90.6647967),(38.6272733,-90.1978889),(32.4104385,-90.16206489),(29.9499323,-90.0701156),(35.1490215,-90.0516285),(39.7989763,-89.6443688),(44.9596017,-89.6298239),(40.6938609,-89.5891008),(43.074761,-89.3837613),(31.3271189,-89.2903392),(42.2713945,-89.093966),(42.7151854,-88.9907743),(39.8628075,-88.89387182),(35.614517,-88.813947),(44.5299412,-88.0248317),(43.0349931,-87.922497),(43.7508284,-87.71453),(41.8755546,-87.6244212),(39.4667025,-87.4139119),(36.5285918,-87.3098696),(30.421309,-87.2169149),(37.7742152,-87.1133304),(40.4258686,-86.9080655),(36.1622257,-86.7743419),(39.1670396,-86.5342881),(43.2341813,-86.2483921),(39.7683795,-86.1580447),(40.4864444,-86.1336351),(42.7876022,-86.1090828),(38.2542376,-85.759407),(42.9633599,-85.6680863),(42.291707,-85.5872286),(36.162839,-85.5016423),(31.228723,-85.40338318),(35.0456297,-85.3096801),(42.3192548,-85.1824269),(41.0799898,-85.1386015),(32.4609764,-84.9877094),(39.8289369,-84.8902383),(42.7337712,-84.5553805),(39.1014537,-84.5124602),(38.0464066,-84.4970393),(33.7490987,-84.3901849),(42.2681569,-83.7312291),(43.0171773,-83.7236024),(41.3747744,-83.6513229),(41.6786754,-83.5127283),(41.9145037,-83.3968252),(33.9595974,-83.376678),(30.8327022,-83.2784851),(39.9622601,-83.0007065),(39.3331197,-82.9824019),(41.4489397,-82.7079605),(35.6009498,-82.5540161),(39.3292396,-82.1012555),(33.4709714,-81.9748429),(32.4487877,-81.7831674),(41.5051613,-81.6934446),(38.3498221,-81.6326301),(34.0007104,-81.0348144),(35.2270869,-80.8431268),(41.0997803,-80.6495194),(36.0998167,-80.2441445),(41.6414438,-80.1514484),(42.129461,-80.085239),(40.4416941,-79.9900861),(39.629526,-79.9558968),(37.270973,-79.9414313),(32.7876012,-79.9402728),(36.0726355,-79.7919754),(34.1987,-79.7677001),(37.40088,-79.18449229),(33.6955167,-78.8817622),(35.0529931,-78.8787058),(42.8864468,-78.8783689),(38.4493315,-78.8688833),(38.029306,-78.4766781),(39.1857762,-78.1631434),(34.2257282,-77.9447107),(43.1854754,-77.61068605),(37.5385087,-77.43428),(34.7540524,-77.4302415),(39.414443,-77.4105783),(38.8949549,-77.0366456),(41.2493292,-77.0027671),(40.2663107,-76.8861122),(39.962398,-76.727392),(39.2908608,-76.6108073),(42.9320202,-76.5672029),(42.4396039,-76.4968019),(38.9786401,-76.492786),(40.03813,-76.3056686),(43.0481221,-76.1474244),(40.335345,-75.9279495),(42.0986867,-75.9179738),(39.952335,-75.163789),(40.7305991,-73.9865812),(42.6511674,-73.754968),(43.3096957,-73.6441045),(42.3604823,-71.0595678),(31.7884104,-81.6357479),(45.1896789,-88.73201312),(36.9751474,-76.3496662),(32.04138105,-102.3327336),(30.0685335,-90.8873185),(47.4071372,-88.2006604),(37.989881,-87.551161),(38.7011797,-84.5057733),(37.5170432,-82.805994),(27.2762419,-82.5592244),(35.287663,-77.6541388),(41.0256428,-75.6046334),(38.3009514,-75.4885315),(46.4083362,-105.8277799),(27.89805,-82.51906),(39.6420215,-80.8564883),(19.5895269,-155.4488688),(24.6670875,-81.5831421),(32.193817,-80.7381637),(38.0729013,-76.0232687),(35.5668467,-75.4684908),(40.795092,-73.4873749),(40.5883246,-75.5640246),(44.2611393,-72.5795127),(32.5084744,-84.9179867),(48.0827524,-123.0846903),(44.507103,-92.91292258),(42.4707245,-118.6421297),(44.0968456,-123.1697524),(41.5225325,-90.5856926),(32.89318052,-117.227146),(44.6415791,-123.1233587),(40.3114316,-74.905776),(40.1138333,-88.2065713),(39.4124,-107.1679),(38.2035967,-92.6266766),(37.648393,-120.9007729),(34.4538383,-117.3907948),(32.791248,-117.1990279),(47.541535,-116.1807249),(39.9795117,-104.8060085),(35.47640095,-97.52947455),(44.009942,-97.5253238),(32.735794,-97.2299579),(32.962456,-96.3343279),(37.91126,-95.1903099),(29.901651,-93.9741989),(40.1648716,-92.1654073),(34.757526,-87.6559009),(33.494923,-86.8494879),(33.570499,-86.7657829),(30.414869,-86.5941599),(40.073597,-85.6630129),(43.7500661,-85.1447325),(35.9295608,-84.1125457),(32.618592,-83.5884269),(28.4922647,-81.2590763),(27.4518379,-80.4211729),(40.854565,-78.2724149),(38.945558,-76.7415049),(40.5703802,-75.4586551),(40.565814,-74.3391209),(41.5409574,-72.7847633),(36.9678224,-120.2654105),(47.6781237,-117.1187889),(45.44923995,-122.715405),(31.218775,-91.179099),(37.7129166,-89.2160546),(33.6659367,-85.8255206),(42.5414947,-83.4004035),(39.5323196,-77.9763876),(41.721626,-72.217319),(41.030666,-83.6550599),(39.092585,-84.8532289),(46.88098595,-96.7851239),(40.508781,-88.9838511),(33.9201654,-78.2342766),(25.7895799,-80.2855704),(46.7442335,-117.1139817),(42.5253897,-92.4413838),(35.7641954,-105.9469662),(30.6262616,-96.3347249),(34.9783294,-111.618724),(40.3161892,-111.6519269),(34.629424,-86.9520296),(29.675779,-95.40215851),(45.8030977,-123.9648582),(32.46858275,-94.72760438),(37.8463451,-81.9917984),(38.8334961,-77.30425582),(42.2923599,-77.1171591),(39.1844444,-75.5427778),(61.235017,-149.845704),(42.264429,-122.784644),(48.492165,-122.650212),(37.758854,-122.440432),(36.234742,-119.622152),(46.240875,-119.220404),(46.408007,-117.035974),(41.173255,-112.004431),(39.103832,-106.560092),(40.562527,-105.067137),(32.924736,-104.356888),(30.280418,-104.071746),(41.88099,-103.672243),(44.624696,-103.327094),(29.368836,-100.89288),(44.573813,-100.36252),(26.246153,-98.195357),(31.066588,-97.613139),(45.28549,-97.550021),(43.58718,-96.763097),(41.260639,-95.93877),(43.932595,-95.543727),(37.21943,-94.908823),(31.609866,-94.650501),(38.750588,-92.208539),(40.821364,-91.406766),(39.934662,-91.378392),(34.620875,-89.410301),(44.050604,-88.572081),(39.490135,-88.275282),(42.642739,-87.861252),(46.415489,-86.93134),(41.716099,-86.530199),(42.071873,-86.14664),(40.142364,-85.508496),(44.028913,-84.60287),(31.578598,-84.156144),(39.826631,-84.017275),(43.556666,-83.994472),(38.427105,-82.560983),(42.973234,-82.427728),(34.839227,-82.384288),(36.952827,-82.342697),(26.632092,-81.949246),(39.984131,-81.787677),(39.355954,-81.522286),(29.884728,-81.318221),(38.598912,-80.758473),(37.220815,-80.445334),(35.888592,-78.822122),(40.420688,-78.660089),(39.935909,-77.655733),(42.01313,-77.36288),(41.316272,-75.758579),(43.123871,-75.428687),(44.717792,-75.008335),(41.997263,-74.386818),(41.269403,-73.885784),(44.688559,-73.469809),(40.8517821,-73.09918527),(42.2664146,-71.7970178),(41.555658,-71.2642153),(42.3709299,-71.1828321),(41.620223,-71.134384),(42.325437,-73.192981),(41.373924,-70.540595),(46.8200431,-67.9230792),(38.831739,-97.624047),(31.156373,-81.48328)]
data = {
"term": term,
"city_name": cityList,
"results": resultSet,
"location": locationSet
}
for city in searchList:
time.sleep(1)
response = urllib2.urlopen('http://' + city + '.craigslist.org/search/web?query=' + term)
html = response.read()
soup = BeautifulSoup(html)
htmldata = soup.find_all(match_class(["paginator"]))#Grabs the line of HTML with all the data
regex= '(\d{3,})'#Try and grab the larger value totalCount
match = re.search(regex, str(htmldata))
# If-statement after search() tests if it succeeded
if match:
result = (match.group(0))#All success goes here
else:
regex= '\s-\s(?P<key>\d+)'#Else defaults to the range (1 to 99)
match = re.search(regex, str(htmldata))
if match:
result = (match.group('key'))
else:
regex= '(\d)'#targeting zero value
match = re.search(regex, str(htmldata))
result = (match.group(0))
print (result)
resultSet.append(result)
#Writes to the file. json.dump is for writing to files, while json.dumps is just fine for working with in memory.
with open(term + '_result.json', 'w') as fp:
json.dump(data, fp, sort_keys=True, indent=4)