python 爬虫，爬取扇贝单词网单词

实现自动分章分页抓取，见 博客：BeautifulSoup + requests 爬取扇贝 python 单词书

from bs4 import BeautifulSoup
from collections import OrderedDict
import requests
import math
import sys

if len(sys.argv) == 2:
    bookUrl = sys.argv[1]
else:
    bookUrl = 'https://www.shanbay.com/wordbook/172933/'

baseUrl = 'https://www.shanbay.com'
head = {'headers':'Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0'}

def getBookName(bookUrl):
    return bookUrl.split('/')[-2]

def getSoup(url):
    rsp = requests.get(url,headers=head,timeout=2)
    if rsp.status_code == 200:
        return BeautifulSoup(rsp.text)
    return None

def getChapter(Soup):
    chapterList = []
    for i in Soup.select('.wordbook-wordlist-name>a'):
        chapterList.append(i['href'])
    return chapterList

def mixChapterUrl(baseUrl,chapterList):
    mixChapterList = []
    for i in chapterList:
        mixChapterList.append(baseUrl+i)
    return mixChapterList

def mixChapterPageUrl(mixChapterList):
    mixChapterPageList = []
    for i in  mixChapterList:
        tmpChapterList = []
        soup = getSoup(i)
        number = int(soup.select('#wordlist-num-vocab')[0].string)
        pages = math.ceil(int(number)/20)
        for j in range(1,pages+1):
            tmpChapterList.append("{}?page={}".format(i,j))
        mixChapterPageList.append(tmpChapterList)
    return mixChapterPageList

def getWord(url):
    soup = getSoup(url)
    word = []
    translate = []
    wordDict = OrderedDict()
    for i in soup.select('.span2>strong'):
        word.append(i.string)
    for i in soup.select('.span10'):
        translate.append(i.string.replace('\n',' '))
    for i in range(len(word)):
        wordDict[word[i]] = translate[i]
    return wordDict

if __name__ == '__main__':

    name = getBookName(bookUrl)
    soup = getSoup(bookUrl)
    chapter = getChapter(soup)
    mixChapterList = mixChapterUrl(baseUrl,chapter)
    word = []
    for i in mixChapterPageUrl(mixChapterList):
        print(len(i))
        for j in i:
            word.append(getWord(j))

    with open(name+'.txt','w') as f:
        for i in word:
            for j in i.keys():
                f.write('[%-20s]\t[%s]\n'%(j,i[j]))

学习了~