基于Python模拟浏览器发送http请求
1.使用urllib2实现
#!/usr/bin/envpython
#-*-coding=utf-8-*-
importurllib2
url="https://www.baidu.com"
req_header={"User-Agent":"Mozilla/5.0(WindowsNT6.1)AppleWebKit/537.11(KHTML,likeGecko)Chrome/23.0.1271.64Safari/537.11",
"Accept":"text/html;q=0.9,*/*;q=0.8",
"Accept-Charset":"ISO-8859-1,utf-8;q=0.7,*;q=0.3",
"Accept-Encoding":"gzip",
"Connection":"close",
"Referer":None#注意如果依然不能抓取的话，这里可以设置抓取网站的host
}
req_timeout=5
req=urllib2.Request(url,None,req_header)
resp=urllib2.urlopen(req,None,req_timeout)
html=resp.read()
print(html)
2.使用requests模块
（1）.get请求
#-*-coding:utf-8-*-
importrequests
url="https://www.baidu.com"
payload={"key1":"value1","key2":"value2"}
r=requests.get(url,params=payload)
printr.text
（2）.post请求
#-*-coding:utf-8-*-
importrequests
url1="http://www.exanple.com/login"#登陆地址
url2="http://www.example.com/main"#需要登陆才能访问的地址
data={"user":"user","password":"pass"}
headers={"Accept":"text/html,application/xhtml+xml,application/xml;",
"Accept-Encoding":"gzip",
"Accept-Language":"zh-CN,zh;q=0.8",
"Referer":"http://www.example.com/",
"User-Agent":"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/42.0.2311.90Safari/537.36"
}
res1=requests.post(url1,data=data,headers=headers)
res2=requests.get(url2,cookies=res1.cookies,headers=headers)
printres2.content#获得二进制响应内容
printres2.raw#获得原始响应内容,需要stream=True
printres2.raw.read(50)
printtype(res2.text)#返回解码成unicode的内容
printres2.url
printres2.history#追踪重定向
printres2.cookies
printres2.cookies["example_cookie_name"]
printres2.headers
printres2.headers["Content-Type"]
printres2.headers.get("content-type")
printres2.json#讲返回内容编码为json
printres2.encoding#返回内容编码
printres2.status_code#返回http状态码
printres2.raise_for_status()#返回错误状态码
（3）.使用session对象的写法
#-*-coding:utf-8-*-
importrequests
s=requests.Session()
url1="http://www.exanple.com/login"#登陆地址
url2="http://www.example.com/main"#需要登陆才能访问的地址
data={"user":"user","password":"pass"}
headers={"Accept":"text/html,application/xhtml+xml,application/xml;",
"Accept-Encoding":"gzip",
"Accept-Language":"zh-CN,zh;q=0.8",
"Referer":"http://www.example.com/",
"User-Agent":"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/42.0.2311.90Safari/537.36"
}
prepped1=requests.Request("POST",url1,
data=data,
headers=headers
).prepare()
s.send(prepped1)
"""
也可以这样写
res=requests.Request("POST",url1,
data=data,
headers=headers
)
prepared=s.prepare_request(res)
#dosomethingwithprepped.body
#dosomethingwithprepped.headers
s.send(prepared)
"""
prepare2=requests.Request("POST",url2,
headers=headers
).prepare()
res2=s.send(prepare2)
printres2.content
"""另一种写法"""
#-*-coding:utf-8-*-
importrequests
s=requests.Session()
url1="http://www.exanple.com/login"#登陆地址
url2="http://www.example.com/main"#需要登陆才能访问的页面地址
data={"user":"user","password":"pass"}
headers={"Accept":"text/html,application/xhtml+xml,application/xml;",
"Accept-Encoding":"gzip",
"Accept-Language":"zh-CN,zh;q=0.8",
"Referer":"http://www.example.com/",
"User-Agent":"Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/42.0.2311.90Safari/537.36"
}
res1=s.post(url1,data=data)
res2=s.post(url2)
print(resp2.content)
3.其他的一些请求方式
>>>r=requests.put("http://httpbin.org/put")
>>>r=requests.delete("http://httpbin.org/delete")
>>>r=requests.head("http://httpbin.org/get")
>>>r=requests.options(http://httpbin.org/get)
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持毛票票。