34 lines
921 B
Python
34 lines
921 B
Python
from bs4 import BeautifulSoup
|
|
import urllib.request
|
|
|
|
base_url = 'http://book.flowingdata.com/vt2/ch3/scraping/'
|
|
|
|
# Header
|
|
print("famindex,AGE,FTOTINC")
|
|
|
|
for i in range(1, 21):
|
|
|
|
# Page URL, based on page number
|
|
page_url = base_url + 'family-income-page'+str(i)+'.html'
|
|
|
|
# Open the page
|
|
req = urllib.request.Request(page_url)
|
|
with urllib.request.urlopen(req) as response:
|
|
|
|
# Save page contents.
|
|
the_page = response.read()
|
|
|
|
# Parse.
|
|
soup = BeautifulSoup(the_page, features="html.parser")
|
|
|
|
# Get the table rows
|
|
rows = soup.find_all("tr")
|
|
|
|
# Save each row of data
|
|
for j in range(1, len(rows)):
|
|
|
|
# Get the cells in a row
|
|
cells = rows[j].findAll("td")
|
|
|
|
# Comma-delimited row
|
|
print(cells[0].string + "," + cells[1].string + "," + cells[2].string) |