第六章 爬虫

第6章 爬虫

1.网络库

• urllib库：http协议常用库

from urllib import request, parse
from io import BytesIO
import gzip

url = 'http://httpbin.org/post'
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, sdch",
    "Accept-Language": "zh-CN,zh;q=0.8",
    "Connection": "close",
    "Cookie": "_gauges_unique_hour=1; _gauges_unique_day=1; _gauges_unique_month=1; _gauges_unique_year=1; _gauges_unique=1",
    "Referer": "http://httpbin.org/",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER"
}
dict = {
    'name': 'value'
}
data = bytes(parse.urlencode(dict), encoding='utf8')
req = request.Request(url=url, data=data, headers=headers, method='POST')
response = request.urlopen(req)
htmls = response.read()
buff = BytesIO(htmls)
f = gzip.GzipFile(fileobj=buff)
print(f.read().decode('utf-8'))

• requests库：http协议常用库

• BeautifulSoup库：xml格式处理库

2.requests库

# get请求
import requests
url = 'http://httpbin.org/get'
data = {'key': 'value', 'abc': 'xyz'}
# .get是使用get方法请求url，字典类型的data不用进行额外处理
response = requests.get(url, data)
print(response.text)	# 打印的为格式化后的数据

# post请求
import requests
url = 'http://httpbin.org/post'
data = {'key': 'value', 'abc': 'xyz'}
# .post表示为post方法
response = requests.post(url, data)
# 返回类型为json格式
print(response.json()) # 打印的数据只有一整行

3.Beautiful Soup的安装和使用

安装命令：pip3 install bs4

赏

谢谢你请我吃炸鸡

支付宝

微信