python 爬小说我梦见了欲望

以下代码由PHP站长网 52php.cn收集自互联网

现在PHP站长网小编把它分享给大家，仅供参考

import urllib.request
import http.cookiejar

import socket
import time
import re


timeout = 20
socket.setdefaulttimeout(timeout)


sleep_download_time = 10 
time.sleep(sleep_download_time)

def makeMyOpener(head = {
    'Connection': 'Keep-Alive','Accept': 'text/html,application/xhtml+xml,*/*','Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3','User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko'
}):
    cj = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
    header = []
    for key,value in head.items():
        elem = (key,value)
        header.append(elem)
    opener.addheaders = header
    return opener

def saveFile(save_path,txts):
    f_obj = open(save_path,'w+')
    for item in txts:
        f_obj.write(item+'n')
    f_obj.close()

#get_code_list
code_list='http://www.kujiang.com/book/catalog/6557'

url_main='http://www.kujiang.com'
oper = makeMyOpener()
uop = oper.open(code_list,timeout=1000)
data = uop.read().decode('utf-8','ignore')

pattern = re.compile('li class="one small-tablet third.*?<a href="(.*?)".*?>(.*?)</a></li>',re.S)

items = re.findall(pattern,data)

print ('获取列表完成')
url_path='url_wmjlyw.txt'

url_r=open(url_path,'r')
url_arr=url_r.readlines(100000)
url_r.close()
print (len(url_arr))




url_file=open(url_path,'a')


print ('获取已下载网址')



i=2
for tmp in items:
    if i > len(items):
        print('game over')
        break
    save_path = tmp[1].replace(' ','')+'.txt'
    url = url_main+tmp[0]
    i=i+1
    if url+'n' in url_arr:
        continue
    print('%d' %i+'写日志：'+url+'n')
    url_file.write(url+'n')
    opene = makeMyOpener()
    op1 = opene.open(url,timeout=1000)
    data = op1.read().decode('utf-8','ignore')
    opene.close()
    pattern = re.compile('<p>　　(.*?)</p>',re.S)
    txts = re.findall(pattern,data)
    saveFile(save_path,txts)
    

url_file.close()

以上内容由PHP站长网【52php.cn】收集整理供大家参考研究

如果以上内容对您有帮助，欢迎收藏、点赞、推荐、分享。

（编辑：李大同）

【声明】本站内容均来自网络，其相关言论仅代表作者个人观点，不代表本站立场。若无意侵犯到您的权利，请及时与联系站长删除相关内容!