教你如何使用Python快速爬取需要的数据

"""例"""
from urllib import request
response = request.urlopen(r'http://bbs.pinggu.org/') 
#返回状态 200证明访问成功
print("返回状态码：  "+str(response.status))
#读取页面信息转换文本并进行解码，如果本身是UTF-8就不要，具体看页面格式
#搜索“charset”查看编码格式
response.read().decode('gbk')[:100]

url = r'http://bbs.pinggu.org/'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
req = request.Request(url, headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')
page[:100]

import urllib.parse
url = r'https://new-api.meiqia.com/v1/throttle/web?api_name=%2Fvisit%2Finit&ent_id=7276&v=1534848690048'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
values={'api_name':'/visit/init','ent_id':'7276','v':'1534848690048'}
data = urllib.parse.urlencode(values).encode(encoding='UTF8')
req = request.Request(url, data,headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')

from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
req = Request("//www.jb51.net /")
try:
    response = urlopen(req)
except HTTPError as e:
    print('服务器无法满足请求.')
    print('错误代码: ', e.code)
except URLError as e:
    print('不能访问服务器.')
    print('原因: ', e.reason)
else:
    print("OK!")
    print(response.read().decode("utf8"))

from urllib.request import Request, urlopen
from urllib.error import  URLError
req = Request("//www.jb51.net /")
try:
    response = urlopen(req)
except URLError as e:
    if hasattr(e, '原因'):
        print('不能访问服务器.')
        print('原因: ', e.reason)
    elif hasattr(e, '代码'):
        print('服务器无法满足请求.')
        print('错误代码: ', e.code)
    else:
        print("OK!")
        print(response.read().decode("utf8"))

import urllib.request
 
# 私密代理授权的账户
user = "user_name"
# 私密代理授权的密码
passwd = "uesr_password"
# 代理IP地址  比如可以使用百度西刺代理随便选择即可
proxyserver = "177.87.168.97:53281"
# 1. 构建一个密码管理对象，用来保存需要处理的用户名和密码
passwdmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
# 2. 添加账户信息，第一个参数realm是与远程服务器相关的域信息，一般没人管它都是写None，后面三个参数分别是 代理服务器、用户名、密码
passwdmgr.add_password(None, proxyserver, user, passwd)
# 3. 构建一个代理基础用户名/密码验证的ProxyBasicAuthHandler处理器对象，参数是创建的密码管理对象
#   注意，这里不再使用普通ProxyHandler类了
proxyauth_handler = urllib.request.ProxyBasicAuthHandler(passwdmgr)
# 4. 通过 build_opener()方法使用这些代理Handler对象，创建自定义opener对象，参数包括构建的 proxy_handler 和 proxyauth_handler
opener = urllib.request.build_opener(proxyauth_handler)
# 5. 构造Request 请求
request = urllib.request.Request("http://bbs.pinggu.org/")
# 6. 使用自定义opener发送请求
response = opener.open(request)
# 7. 打印响应内容
print (response.read())

pip install BeautifulSoup

# 想要抓取我们需要的东西需要进行定位，寻找到标志
from bs4 import BeautifulSoup
soup = BeautifulSoup('<meta content="all" name="robots" />',"html.parser")
tag=soup.meta

# tag的类别
type(tag)
>>> bs4.element.Tag

# tag的name属性
tag.name
>>> 'meta'

# attributes属性
tag.attrs
>>> {'content': 'all', 'name': 'robots'}

# BeautifulSoup属性
type(soup)
>>> bs4.BeautifulSoup
soup.name
>>> '[document]'

# 字符串的提取
markup='<b><a href="http://house.people.com.cn/" rel="external nofollow"  target="_blank">房产</a></b>'
soup=BeautifulSoup(markup,"lxml")
text=soup.b.string
text
>>> '房产'
type(text)
>>> bs4.element.NavigableString

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")

#通过页面解析得到结构数据进行处理
from bs4 import BeautifulSoup
soup=BeautifulSoup(html.text,"lxml")

#定位
lptable = soup.find('table',width='780')

# 解析
for i in lptable.find_all("td",width="680"):    
    title = i.b.strong.a.text
    href = "http://www.cwestc.com"+i.find('a')['href']
#     href = i.find('a')['href']
    date = href.split("/")[4]
    print (title,href,date)

from lxml import etree

html="""
<!DOCTYPE html>
<html>
<head lang="en">
    <title>test</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<div id="content">
    <ul id="ul">
        <li>NO.1</li>
        <li>NO.2</li>
        <li>NO.3</li>
    </ul>
    <ul id="ul2">
        <li>one</li>
        <li>two</li>
    </ul>
</div>
<div id="url">
    <a href="http://www.crossgate.com" rel="external nofollow"  title="crossgate">crossgate</a>
    <a href="http://www.pinggu.org" rel="external nofollow"  title="pinggu">pinggu</a>
</div>
</body>
</html>
"""

#这里使用id属性来定位哪个div和ul被匹配 使用text()获取文本内容
selector=etree.HTML(html)
content=selector.xpath('//div[@id="content"]/ul[@id="ul"]/li/text()') 
for i in content:
    print (i)

#这里使用//从全文中定位符合条件的a标签，使用“@标签属性”获取a便签的href属性值
con=selector.xpath('//a/@href') 
for i in con:
    print (i)

#使用绝对路径 #使用相对路径定位 两者效果是一样的
con=selector.xpath('/html/body/div/a/@title') 
print (len(con))
print (con[0],con[1])

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=1"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://news.cqcoal.com/blank/nl.jsp?tid=238"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text

import urllib
import urllib.request
import requests

url = "http://news.cqcoal.com/manage/newsaction.do?method:webListPageNewsArchivesByTypeid"
post_param = {'pageNum':'1',\
              'pageSize':'20',\
              'jsonStr':'{"typeid":"238"}'}
return_data = requests.post(url,data =post_param)
content=return_data.text
content