-
Notifications
You must be signed in to change notification settings - Fork 0
/
final21.py
75 lines (57 loc) · 2.73 KB
/
final21.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#Get the first page to extract number of pages
import requests
from bs4 import BeautifulSoup
import pandas
r=requests.get("http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/",
headers={'User-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'}) # We use the headers here
c=r.content
soup=BeautifulSoup(c,"html.parser")
all=soup.find_all("div",{"class":"propertyRow"})
all[0].find("h4",{"class":"propPrice"}).text.replace("\n","").replace(" ","")
page_nr=soup.find_all("a",{"class":"Page"})[-1].text
#last index gives value of last page
#print(page_nr,"number of pages were found")
l=[]
base_url = "http://www.pyclass.com/real-estate/rock-springs-wy/LCWYROCKSPRINGS/t=0&s=" #s=0:firstpage,s=10:secondpage etc.
for page in range(0,int(page_nr)*10,10): #traverses through pages
print(base_url+str(page)+".html")
r = requests.get(base_url+str(page)+".html",
headers={'User-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0'})
c = r.content
soup = BeautifulSoup(c,"html.parser")
all = soup.find_all("div",{"class":"propertyRow"})
for item in all: #gets data out of each page
d={}
d["Address"]=item.find_all("span",{"class","propAddressCollapse"})[0].text
try:
d["Locality"]=item.find_all("span",{"class","propAddressCollapse"})[1].text #address is in two lines
except:
d["Locality"]=None
d["Price"]=item.find("h4",{"class","propPrice"}).text.replace("\n","").replace(" ","")
try:
d["Beds"]=item.find("span",{"class","infoBed"}).find("b").text #.text cannot be applied to None values;so passes
except:
d["Beds"]=None
try:
d["Area"]=item.find("span",{"class","infoSqFt"}).find("b").text
except:
d["Area"]=None
try:
d["Full Baths"]=item.find("span",{"class","infoValueFullBath"}).find("b").text
except:
d["Full Baths"]=None
try:
d["Half Baths"]=item.find("span",{"class","infoValueHalfBath"}).find("b").text
except:
d["Half Baths"]=None
#loop for finding loop size which all props dont have
for column_group in item.find_all("div",{"class":"columnGroup"}):
#print(column_group)
for feature_group,feature_name in zip(column_group.find_all("span",{"class":"featureGroup"}),column_group.find_all("span",{"class":"featureName"})):
#print(feature_group.text,feature_name.text)
if "Lot Size" in feature_group.text:
d["Lot Size"]=feature_name.text
l.append(d)
df=pandas.DataFrame(l)
#print(df)
df.to_csv("Data.csv")