python项目策划书_Python实战计划学习笔记0630

实战计划第三天，抓了300条数据。

最终成果是这样的：

Paste_Image.png

我的代码：

#!/usr/bin/env python #告诉计算机执行程序在系统环境变量中的名字，详细位置在环境变量中设置好了

#-*- coding: utf-8 -*-

from bs4 import BeautifulSoup

import time

import requests

urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i))for i in range(1,10,1)]

def house_source(url,data=None):

wb_data = requests.get(url)

time.sleep(1)

soup = BeautifulSoup(wb_data.text, 'lxml')

titles = soup.select("body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > h4 > em")

adresses = soup.select('body > div.wrap.clearfix.con_bg > div.con_l > div.pho_info > p > span.pr5')

prices = soup.select('#pricePart > div.day_l > span')

imgs = soup.select('img[id="curBigImage"]')

names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')

picts = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')

males = soup.select('div[class="member_ico1"]')

for title, adress, price, img, name, pict, male in zip(titles, adresses, prices, imgs, names, picts, males):

data = {

'title': title.get_text(),

'adress': adress.get_text(),

'price': price.get_text(),

'img': img.get('src'),

'name': name.get_text(),

'pict': pict.get('src'),

'male': get_lorder_male(male.get('class')) #写个函数处理

}

print(data)

def get_links(url):

wb_data = requests.get(url)

time.sleep(2)

soup = BeautifulSoup(wb_data.text, 'lxml')

links = soup.select("#page_list > ul > li > a" )

for link in links:

href = link.get('href')

house_source(href)

def get_lorder_male(class_name):

if class_name == ['member_ico']: # 判断语句

return '男'

else:

return '女'

for single_url in urls:

get_links(single_url)

总结

format(str(i))for i in range(1,10,1) 找网页编号规律

构建字典时对key的处理，如get到css样式对应的属性male.get('class')，link.get('href')，

函数的构建 如def get_lorder_male(class_name):

if class_name == ['member_ico']: # 判断语句 return '男'