python爬取网页数据步骤,python爬取网页详细教程_python获取网页信息

>>> kv = {'key1':'value1','key2':'value2'}
>>> r = requests.request('GET','http://python123.io/ws',params=kv)
>>> print(r.url)
https://python123.io/ws?key1=value1&key2=value2

>>> kv = {'key1':'value1','key2':'value2'}
>>> r = requests.request('POST','http://python123.io/ws',data=kv)
>>> body = '主题内容'
>>> r = requests.request('POST','http:///python123.io/ws',data=body)

>>> kv = {'key1':'value1','key2':'value2'}
>>> r = requests.request('POST','http://python123.io/ws',json=kv)

>>> hd = {'user-agent':'Chrome/10'}
>>> r = requests.request('POST','http://www.baidu.com',headers=hd)

>>> f = {'file':open('/root/po.sh','rb')}
>>> r = requests.request('POST','http://python123.io/ws',file=f)

>>> r = requests.request('GET','http://python123.io/ws',timeout=30)

>>> pxs = {'http':'http://user:pass@10.10.10.2:1234',
... 'https':'https://10.10.10.3:1234'}
>>> r = requests.request('GET','http://www.baidu.com',proxies=pxs)

import requests
def getHTMLText(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

if __name__ == "__main__":
    url = "http://www.baidu.com"
    print(getHTMLText(url))

import requests
url = 'https://item.jd.com/5089267.html'
try:
    r = requests.get(url)
    r.raise_for_status()
    r.encoding =r.apparent_encoding
    print(r.text[:1000])
except:
    print("爬取失败")

import requests
url = 'http://product.dangdang.com/26487763.html'
try:
    r = requests.get(url)
    r.raise_for_status()
    r.encoding =r.apparent_encoding
    print(r.text[:1000])
except IOError as e:
    print(str(e))

import requests
url = 'http://product.dangdang.com/26487763.html'
try:
    kv = {'user-agent':'Mozilla/5.0'}
    r = requests.get(url,headers=kv)
    r.raise_for_status()
    r.encoding =r.apparent_encoding
    print(r.text[:1000])
except IOError as e:
    print(str(e))

import requests
keyword = "python"
try:
    kv = {'wd':keyword}
    r = requests.get("http://www.baidu.com/s",params=kv)
    print(r.request.url)
    r.raise_for_status()
    print(len(r.text))
except IOError as e:
    print(str(e))

import requests
keyword = "Linux"
try:
    kv = {'q':keyword}
    r = requests.get("http://www.so.com/s",params=kv)
    print(r.request.url)
    r.raise_for_status()
    print(len(r.text))
except IOError as e:
    print(str(e))

import requests
import os
url = "http://www.xiaohuar.com/d/file/20141116030511162.jpg"
dir = "D://pics//"
path = dir + url.split('/')[-1] #设置图片保存路径并以原图名名字命名
try:
    if not os.path.exists(dir):
        os.mkdir(dir)
    if not os.path.exists(path):
        r = requests.get(url)
        with open(path,'wb') as f:
            f.write(r.content)
            f.close()
            print("文件保存成功")
    else:
        print("文件已存在")
except IOError as e:
    print(str(e))

import requests
url = "http://www.ip38.com/ip.php?ip="
try:
    r = requests.get(url+'104.193.88.77')
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text)
except IOError as e:
    print(str(e))

import requests
import json

def get_translate_date(word=None):
    url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule"
    #post参数需要放在请求实体里，构建一个新字典
    form_data = {'i': word,
                 'from': 'AUTO',
                 'to': 'AUTO',
                 'smartresult': 'dict',
                 'client': 'fanyideskweb',
                 'salt': '15569272902260',
                 'sign': 'b2781ea3e179798436b2afb674ebd223',
                 'ts': '1556927290226',
                 'bv': '94d71a52069585850d26a662e1bcef22',
                 'doctype': 'json',
                 'version': '2.1',
                 'keyfrom': 'fanyi.web',
                 'action': 'FY_BY_REALTlME'
                 }
    #请求表单数据
    response = requests.post(url,data=form_data)
    #将JSON格式字符串转字典
    content = json.loads(response.text)
    #打印翻译后的数据
    print(content['translateResult'][0][0]['tgt'])

if __name__ == '__main__':
    word = input("请输入你要翻译的文字：")
    get_translate_date(word)

from bs4 import BeautifulSoup
soup = BeautifulSoup('<p>Hello</p>','lxml')
print(soup.p.string)

>>> from bs4 import BeautifulSoup
>>> import requests
>>> r = requests.get("http://python123.io/ws/demo.html")
>>> demo = r.text
>>> demo
'<html><head><title>This is a python demo page</title></head>\r\n<body>\r\n<p class="title"><b>The demo python introduces several python courses.</b></p>\r\n<p class="course">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\r\n<a href="http://www.icourse163.org/course/BIT-268001" class="py1" id="link1">Basic Python</a> and <a href="http://www.icourse163.org/course/BIT-1001870001" class="py2" id="link2">Advanced Python</a>.</p>\r\n</body></html>'
>>> soup = BeautifulSoup(demo,"html.parser")
>>> soup.title #获取标题
<title>This is a python demo page</title>
>>> soup.a #获取a标签
<a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python</a>
>>> soup.title.string
'This is a python demo page'
>>> soup.prettify() #输出html标准格式内容
'<html>\n <head>\n <title>\n This is a python demo page\n </title>\n </head>\n <body>\n <p class="title">\n <b>\n The demo python introduces several python courses.\n </b>\n </p>\n <p class="course">\n Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n <a class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">\n Basic Python\n </a>\n and\n <a class="py2" href="http://www.icourse163.org/course/BIT-1001870001" id="link2">\n Advanced Python\n </a>\n .\n </p>\n </body>\n</html>'
>>> soup.a.name #每个<tag>都有自己的名字，通过<tag>.name获取
'a'
>>> soup.p.name
'p'
>>> tag = soup.a
>>> tag.attrs
{'href': 'http://www.icourse163.org/course/BIT-268001', 'class': ['py1'], 'id': 'link1'}
>>> tag.attrs['class']
['py1']
>>> tag.attrs['href']
'http://www.icourse163.org/course/BIT-268001'
>>> type(tag.attrs)
<class 'dict'>
>>> type(tag)
<class 'bs4.element.Tag'>
>>>

from bs4 import BeautifulSoup
import requests
demo = requests.get("http://python123.io/ws/demo.html").text
soup = BeautifulSoup(demo,"html.parser")
#标签树的上行遍历
print("遍历儿子节点：\n")
for child in soup.body.children:
 print(child)
 
print("遍历子孙节点：\n")
for child1 in soup.body.descendants:
 print(child1)
 
print(soup.title.parent)
print(soup.html.parent)
for parent in soup.a.parents:
 if parent is None:
 print(parent)
 else:
 print(parent.name)
#标签树的平行遍历
print(soup.a.next_sibling)
print(soup.a.next_sibling.next_sibling)
print(soup.a.previous_sibling)

#!/usr/bin/python3
import re
#替换
phone = '18898537584 #这是我的电话号码'
print('我的电话号码:',re.sub('#.*','',phone)) #去掉注释
print(re.sub('\D','',phone))
#search
ip_addr = re.search('(\d{3}\.){1,3}\d{1,3}\.\d{1,3}',os.popen('ifconfig').read())
print(ip_addr)
#match
>>> a = re.match('\d+','2ewrer666dad3123df45')
>>> print(a.group())
2 

>>> import re
>>> print(re.search(r'(\d+)-([a-z])','34324-dfsdfs777-hhh').group(0)) #返回整体
34324-d
>>> print(re.search(r'(\d+)-([a-z])','34324-dfsdfs777-hhh').group(1))  #返回第一组
34324
>>> print(re.search(r'(\d+)-([a-z])','34324-dfsdfs777-hhh').group(2))  #获取第二组
d
>>> print(re.search(r'(\d+)-([a-z])','34324-dfsdfs777-hhh').group(3))  #不存在。报错“no such group”
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
IndexError: no such group

7.4 re.compile 函数

compile 函数用于编译正则表达式，生成一个正则表达式（ Pattern ）对象。语法格式：
re.compile(pattern[, flags])
参数：
pattern : 一个字符串形式的正则表达式
flags : 可选，表示匹配模式，比如忽略大小写，多行模式等，具体参数为：
re.I 忽略大小写
re.L 表示特殊字符集 \w, \W, \b, \B, \s, \S 依赖于当前环境
re.M 多行模式
re.S 即为 . 并且包括换行符在内的任意字符（. 不包括换行符）
re.U 表示特殊字符集 \w, \W, \b, \B, \d, \D, \s, \S 依赖于 Unicode 字符属性数据库
re.X 为了增加可读性，忽略空格和 # 后面的注释
常用的是re.I和re.S

>>> import re
>>> pattern = re.compile('\d+',re.S)  #用于匹配至少一个数字
>>> res = re.findall(pattern,"my phone is 18898566588")
>>> print(res)
['18898566588']

import json
import requests
from requests.exceptions import RequestException #引入异常
import re
import time
def get_one_page(url):
    try:
        response = requests.get(url)
        if response.status_code == 200: #由状态码判断返回结果
            return response.text #返回网页内容
        return None
    except RequestException:
        return None

def parse_one_page(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
                         + '.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
                         + '.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S) #compile函数表示一个方法对象，re.s匹配多行
    items = re.findall(pattern, html) #以列表形式返回全部能匹配的字符串。
    for item in items:  #将结果以字典形式返回键值对
        yield {   #把这个方法变成一个生成器
            'index': item[0],
            'image': item[1],
            'title': item[2],
            'actor': item[3].strip()[3:],
            'time': item[4].strip()[5:],
            'score': item[5] + item[6]  #将评分整数和小数结合起来
        }

def write_to_file(content):
    with open('result.txt', 'a', encoding='utf-8') as f:  #将结果写入文件
        f.write(json.dumps(content, ensure_ascii=False) + '\n')

def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    html = get_one_page(url)
    for item in parse_one_page(html):
        print(item)
        write_to_file(item)

if __name__ == '__main__':
    for i in range(10):
        main(offset=i * 10)
        time.sleep(1)

import scrapy
class DoubanmovietopItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #排名
    ranking = scrapy.Field()
    #电影名称
    movie_name = scrapy.Field()
    #评分
    score = scrapy.Field()
    #评论人数
    score_num = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from DoubanMovieTop.items import DoubanmovietopItem

class DoubanSpider(scrapy.Spider):
    name = 'douban'
    #allowed_domains = ['movie.douban.com']
    def start_requests(self):
        start_urls = 'https://movie.douban.com/top250'
        yield scrapy.Request(start_urls)

    def parse(self, response):
        item = DoubanmovietopItem()
        movies = response.xpath('//ol[@class="grid_view"]/li')
        for movie in movies:
            item['ranking'] = movie.xpath('.//div[@class="pic"]/em/text()').extract()[0]
            item['movie_name'] = movie.xpath('.//div[@class="hd"]/a/span[1]/text()').extract()[0]
            item['score'] = movie.xpath('.//div[@class="star"]/span[@class="rating_num"]/text()').extract()[0]
            item['score_num'] = movie.xpath('.//div[@class="star"]/span/text()').re(r'(\d+)人评价')[0]  #Selector也有一种.re()
            yield item
        next_url = response.xpath('//span[@class="next"]/a/@href').extract()
        if next_url:
            next_url = 'https://movie.douban.com/top250' + next_url[0]
            yield scrapy.Request(next_url)

scrapy crawl douban -o douban.csv