Week13-1_web_scraping

Verify the Requests installation

import requests               

url = 'https://news.daum.net/'  
response = requests.get(url)    
html_text = response.text
print(html_text[:500])

Verify the Beautifulsoup installation

from bs4 import BeautifulSoup as bs

html = bs(html_text, 'html.parser')
print(html.select('link'))

select()

html_hyperlink_list = html.select('a')
sample = html_hyperlink_list[5]
print(sample)
print(sample.text)
print(sample['class'])

Find a tag inside a tag

html_list = html.select('link script')
print(html_list[0])

Find tags by class

html_list_by_class = html.select('.item_issue')
print(html_list_by_class[0])

Find tags using if

html_list_by_id = html.select('#mainContent')
print(html_list_by_id[0])

Find news headline patterns

headlines = html.select('.item_issue .tit_g a')
print(headlines[0])

Extract text from a tag

for headline in headlines:
  print(headline.text)

Remove unnecessary line breaks from text

for headline in headlines:
  print(headline.text.replace('\n', ''))

Remove additional unnecessary spaces from text

for headline in headlines:
  print(headline.text.replace('\n', '').replace('  ', ''))

###Complete news headline scraping with numbered headlines

headlines = html.select('.item_issue .tit_g a')
for i, headline in enumerate(headlines):
  headline_text = headline.text.replace('\n', '').replace('  ', '')
  print(i, headline_text)

Scrape Korea Meteorological Administration weather data

url = 'https://www.weather.go.kr/w/obs-climate/land/city-obs.do'  
response = requests.get(url)    
html_text = response.text
html = bs(html_text, 'html.parser')
cities = html.select('tbody tr')

temperatures = {}
for city in cities:
  city_name = city.select('a')[0].text
  temp = city.select('td')[5].text
  temperatures[city_name] = temp
print(temperatures)