Week13-1_web_scraping

Requests 정상 설치 확인

import requests               

url = 'https://news.daum.net/'  
response = requests.get(url)    
html_text = response.text
print(html_text[:500])

Beautifulsoup 정상 설치 확인

from bs4 import BeautifulSoup as bs

html = bs(html_text, 'html.parser')
print(html.select('link'))

select()

html_hyperlink_list = html.select('a')
sample = html_hyperlink_list[5]
print(sample)
print(sample.text)
print(sample['class'])

tag 내의 tag 찾기

html_list = html.select('link script')
print(html_list[0])

class로 tag 찾기

html_list_by_class = html.select('.item_issue')
print(html_list_by_class[0])

if로 tag 찾기

html_list_by_id = html.select('#mainContent')
print(html_list_by_id[0])

뉴스 헤드라인 패턴 찾기

headlines = html.select('.item_issue .tit_g a')
print(headlines[0])

tag에서 text 분리하기

for headline in headlines:
  print(headline.text)

text에서 불필요한 줄바꿈 없애기

for headline in headlines:
  print(headline.text.replace('\n', ''))

text에서 추가로 불필요한 띄어쓰기 없애기

for headline in headlines:
  print(headline.text.replace('\n', '').replace('  ', ''))

###번호를 매겨 뉴스 헤드라인 스크래핑 완료하기

headlines = html.select('.item_issue .tit_g a')
for i, headline in enumerate(headlines):
  headline_text = headline.text.replace('\n', '').replace('  ', '')
  print(i, headline_text)

기상청 날씨 스크랩

url = 'https://www.weather.go.kr/w/obs-climate/land/city-obs.do'  
response = requests.get(url)    
html_text = response.text
html = bs(html_text, 'html.parser')
cities = html.select('tbody tr')

temperatures = {}
for city in cities:
  city_name = city.select('a')[0].text
  temp = city.select('td')[5].text
  temperatures[city_name] = temp
print(temperatures)