Python 爬蟲-模擬登入知乎-爬取拉勾網職位資訊

阿新 • • 發佈：2019-01-16

用Python寫爬蟲是很方便的,最近看了xlzd.me的文章，他的文章寫的很到位，提供了很好的思路。因為他的文章部分程式碼省略了。下面是基於他的文章的三個程式碼片段:
基於Python3,Python2的話需要修改下input輸入函式和print的用法。

爬取豆瓣電影top250
爬取拉勾網職位資訊
模擬登陸知乎
為什麼沒人給我點贊。？！

有些程式碼做了更改。其中把獲取的資料儲存到excel中。關於存取資料到excel可以看我的另一篇文章：。

用到的庫

requests
Beautiful Soup
openpyxl

1. 爬取豆瓣電影top250，存到excel表格中

#!/usr/bin/env python
# encoding=utf-8
import requests,re
import codecs
from bs4 import BeautifulSoup
from openpyxl import Workbook
wb = Workbook()
dest_filename = '電影.xlsx'
ws1 = wb.active  
ws1.title = "電影top250"

DOWNLOAD_URL = 'http://movie.douban.com/top250/'

def download_page(url):
    """獲取url地址頁面內容"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }
    data = requests.get(url, headers=headers).content
    return data


def get_li(doc):
    soup = BeautifulSoup(doc, 'html.parser')
    ol = soup.find('ol', class_='grid_view')
    name = [] #名字
    star_con = [] #評價人數
    score = []  #評分
    info_list = []  #短評
    for i in ol.find_all('li'):
        detail = i.find('div', attrs={'class': 'hd'})
        movie_name = detail.find('span', attrs={'class': 'title'}).get_text() #電影名字
        level_star = i.find('span',attrs={'class':'rating_num'}).get_text() #評分
        star = i.find('div',attrs={'class':'star'})
        star_num = star.find(text=re.compile('評價'))  #評價

        info = i.find('span',attrs={'class':'inq'})  #短評
        if info:     #判斷是否有短評
            info_list.append(info.get_text())
        else:
            info_list.append('無')
        score.append(level_star)
        

        name.append(movie_name)
        star_con.append(star_num)
    page = soup.find('span', attrs={'class': 'next'}).find('a') #獲取下一頁
    if page:
        return name,star_con,score,info_list,DOWNLOAD_URL + page['href']
    return name,star_con,score,info_list,None


def main():
    url = DOWNLOAD_URL
    name = []
    star_con=[]
    score = []
    info = []
    while url:
        doc = download_page(url)
        movie,star,level_num,info_list,url = get_li(doc)
        name = name + movie
        star_con = star_con + star
        score = score+level_num
        info = info+ info_list
    for (i,m,o,p) in zip(name,star_con,score,info):
        col_A = 'A%s'%(name.index(i)+1)
        col_B = 'B%s'%(name.index(i)+1)
        col_C = 'C%s'%(name.index(i)+1)
        col_D = 'D%s'%(name.index(i)+1)
        ws1[col_A]=i
        ws1[col_B] = m
        ws1[col_C] = o
        ws1[col_D] = p
    wb.save(filename=dest_filename)

if __name__ == '__main__':
    main()

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879

#!/usr/bin/env python# encoding=utf-8import requests,reimport codecsfrom bs4 import BeautifulSoupfrom openpyxl import

Workbookwb=Workbook()dest_filename='電影.xlsx'ws1=wb.active ws1.title="電影top250"DOWNLOAD_URL='http://movie.douban.com/top250/'def download_page(url):"""獲取url地址頁面內容"""headers={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'}data

=requests.get(url,headers=headers).contentreturndatadef get_li(doc):soup=BeautifulSoup(doc,'html.parser')ol=soup.find('ol',class_='grid_view')name=[]#名字star_con=[]#評價人數score=[]#評分info_list=[]#短評foriinol.find_all('li'):detail=i.find('div',attrs={'class':'hd'})movie_name=detail.find('span',attrs={'class':'title'}).get_text()#電影名字level_star=i.find('span',attrs={'class':'rating_num'}).get_text()#評分star=i.find('div',attrs={'class':'star'})star_num=star.find(text=re.compile('評價'))#評價info=i.find('span',attrs={'class':'inq'})#短評ifinfo:#判斷是否有短評info_list.append(info.get_text())else:info_list.append('無')score.append(level_star)name.append(movie_name)star_con.append(star_num)page=soup.find('span',attrs={'class':'next'}).find('a')#獲取下一頁ifpage:returnname,star_con,score,info_list,DOWNLOAD_URL+page['href']returnname,star_con,score,info_list,Nonedef main():url=DOWNLOAD_URLname=[]star_con=[]score=[]info=[]whileurl:doc=download_page(url)movie,star,level_num,info_list,url=get_li(doc)name=name+moviestar_con=star_con+starscore=score+level_numinfo=info+info_listfor(i,m,o,p)inzip(name,star_con,score,info):col_A='A%s'%(name.index(i)+1)col_B='B%s'%(name.index(i)+1)col_C='C%s'%(name.index(i)+1)col_D='D%s'%(name.index(i)+1)ws1[col_A]=iws1[col_B]=mws1[col_C]=ows1[col_D]=pwb.save(filename=dest_filename)if__name__=='__main__':main()

結果如下：

2. 爬取拉勾網Python職位資訊

職位資訊儲存在json中，獲取到json物件，再從中遍歷出公司名、地址、待遇等資訊。

Python

import requests
from openpyxl import Workbook

def get_json(url, page, lang_name):
    data = {'first': 'true', 'pn': page, 'kd': lang_name}
    json = requests.post(url, data).json()
    list_con = json['content']['positionResult']['result']
    info_list = []
    for i in list_con:
        info = []
        info.append(i['companyShortName'])
        info.append(i['companyName'])
        info.append(i['salary'])
        info.append(i['city'])
        info.append(i['education'])
        info_list.append(info)
    return info_list


def main():
    lang_name = input('職位名：')
    page = 1
    url = 'http://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
    info_result = []
    while page < 31:
        info = get_json(url, page, lang_name)
        info_result = info_result + info
        page += 1
    wb = Workbook()
    ws1 = wb.active
    ws1.title = lang_name
    for row in info_result:
        ws1.append(row)
    wb.save('職位資訊.xlsx')

if __name__ == '__main__':
    main()

12345678910111213141516171819202122232425262728293031323334353637

importrequestsfromopenpyxl importWorkbookdefget_json(url,page,lang_name):data={'first':'true','pn':page,'kd':lang_name}json=requests.post(url,data).json()list_con=json['content']['positionResult']['result']info_list=[]foriinlist_con:info=[]info.append(i['companyShortName'])info.append(i['companyName'])info.append(i['salary'])info.append(i['city'])info.append(i['education'])info_list.append(info)returninfo_listdefmain():lang_name=input('職位名：')page=1url='http://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'info_result=[]whilepage<31:info=get_json(url,page,lang_name)info_result=info_result+infopage+=1wb=Workbook()ws1=wb.activews1.title=lang_nameforrow ininfo_result:ws1.append(row)wb.save('職位資訊.xlsx')if__name__=='__main__':main()

執行結果：

3. 模擬登入知乎

通過開發者工具，獲取post的資料。

import requests,time
from bs4 import BeautifulSoup
def get_captcha(data):
    with open('captcha.gif','wb') as fp:
        fp.write(data)
    return input('輸入驗證碼：')

def login(username,password,oncaptcha):
    sessiona = requests.Session()
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'}
    _xsrf = BeautifulSoup(sessiona.get('https://www.zhihu.com/#signin',headers=headers).content,'html.parser').find('input',attrs={'name':'_xsrf'}).get('value')
    captcha_content = sessiona.get('https://www.zhihu.com/captcha.gif?r=%d&type=login'%(time.time()*1000),headers=headers).content
    data = {
        "_xsrf":_xsrf,
        "email":username,
        "password":password,
        "remember_me":True,
        "captcha":oncaptcha(captcha_content)
    }
    resp = sessiona.post('https://www.zhihu.com/login/email',data,headers=headers).content
    print(resp)
    return resp 

if __name__ == "__main__":
    login('your_email','your_password',get_captcha)

12345678910111213141516171819202122232425

import requests,timefrom bs4 import BeautifulSoupdef get_captcha(data):with open('captcha.gif','wb')asfp:fp.write(data)returninput('輸入驗證碼：')def login(username,password,oncaptcha):sessiona=requests.Session()headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'}_xsrf=BeautifulSoup(sessiona.get('https://www.zhihu.com/#signin',headers=headers).content,'html.parser').find('input',attrs={'name':'_xsrf'}).get('value')captcha_content=sessiona.get('https://www.zhihu.com/captcha.gif?r=%d&type=login'%(time.time()*1000),headers=headers).contentdata={"_xsrf":_xsrf,"email":username,"password":password,"remember_me":True,"captcha":oncaptcha(captcha_content)}resp=sessiona.post('https://www.zhihu.com/login/email',data,headers=headers).contentprint(resp)returnresp if__name__=="__main__":login('your_email','your_password',get_captcha)

執行後會在執行目錄下得到驗證碼圖片：

輸入驗證碼後得到如下響應結果表明登入成功。

Python 爬蟲-模擬登入知乎-爬取拉勾網職位資訊

1. 爬取豆瓣電影top250，存到excel表格中

2. 爬取拉勾網Python職位資訊

3. 模擬登入知乎

Python 爬蟲-模擬登入知乎-爬取拉勾網職位資訊

【圖文詳解】scrapy爬蟲與動態頁面——爬取拉勾網職位資訊（1）

【圖文詳解】scrapy爬蟲與動態頁面——爬取拉勾網職位資訊（2）

Scrapy爬取拉勾網職位資訊

python爬蟲——模擬登入教務系統爬取成績

爬蟲學習之17：爬取拉勾網網招聘資訊（非同步載入+Cookie模擬登陸）

python爬蟲: 爬取拉勾網職位並分析

用python爬取拉勾網招聘資訊並以CSV檔案儲存

Python爬取拉勾網招聘資訊存入資料庫

Python爬取拉勾網招聘資訊

Python爬取拉勾網招聘資訊並可視化分析

Python scrapy 爬取拉勾網招聘資訊

HttpClient爬取拉勾網招聘資訊

scrapy抓取拉勾網職位資訊（一）——scrapy初識及lagou爬蟲專案建立

scrapy抓取拉勾網職位資訊（四）——對欄位進行提取

python爬蟲模擬登陸知乎網

Python爬蟲：爬取拉勾網資料分析崗位資料

python爬蟲學習筆記分析Ajax爬取果殼網文章

python爬蟲(16)使用scrapy框架爬取頂點小說網

Python 爬蟲入門-爬取拉勾網實戰

Python 爬蟲-模擬登入知乎-爬取拉勾網職位資訊

1. 爬取豆瓣電影top250，存到excel表格中

2. 爬取拉勾網Python職位資訊

3. 模擬登入知乎

相關推薦