20.python-爬虫的基础认知，urllib中的get和post请求

本文链接：https://blog.csdn.net/Programmer_huangtao/article/details/88536076

总之访问时，差不多是一样的，只是多了些参数，访问会更安全些

from urllib import request,parse
# url = 'http://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
# res = request.urlopen(url)
# print(res.read().decode('utf-8'))
#存在异常行为，需要登录后才能继续访问,没有请求头，被确认为爬虫，被禁，假数据
# url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false'
# header = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
# }
# data = {
#         'first': 'true',
#         'pn': 1,
#         'kd': 'python爬虫'
# }
# req = request.Request(url,headers=header,data=parse.urlencode(data).encode('utf-8'),method='POST')
# res = request.urlopen(req)
# print(res.read().decode('utf-8'))

#{"status":false,"msg":"您操作太频繁,请稍后再访问","clientIp":"223.21.30.168","state":2402},套路

url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult=false'
headers = {
    # 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
    'Referer': 'https://www.lagou.com/jobs/list_python%E7%88%AC%E8%99%AB?oquery=python&fromSearch=true&labelWords=relative&city=%E5%8C%97%E4%BA%AC',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}
data = {
        'first': 'true',
        'pn': 1,
        'kd': 'python'
}
req = request.Request(url,headers=headers,data=parse.urlencode(data).encode('utf-8'),method='POST')
res = request.urlopen(req)
print(res.read().decode('utf-8'))