开源软件批量下载

以下代码由PHP站长网 52php.cn收集自互联网

现在PHP站长网小编把它分享给大家，仅供参考

import requests
import lxml
import gevent
from lxml import html
import os

from gevent import monkey; monkey.patch_os()
monkey.patch_socket()


from multiprocessing import Process



#view-source:https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/
base_url = "https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/epel/"
url_send="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/"
repodata_url ="https://repos.fedorapeople.org/repos/openstack/openstack-icehouse/epel-6/repodata/"

puppetlab = "http://yum.puppetlabs.com/el/6/products/x86_64/"



def get_rpms(url):

    r = requests.get( url )
    doc = html.fromstring(r.text)
    doc.make_links_absolute( url )
    links = doc.iterlinks()
    rpms = [i[2]  for i in links if i[2].endswith("rpm")]
    return rpms

def  get_gzs(url):
    r = requests.get(url)
    doc = html.fromstring(r.text)
    doc.make_links_absolute(url)
    links = doc.iterlinks()
    rpms = [i[2] for i  in links if i[2].endswith("gz") or i[2].endswith("bz2")]
    return rpms


def download(url,dir):
    r = requests.get( url )
    filename = url.split("/")[-1]
    print(filename)
    f = open( os.path.join(dir,filename),"wb")
    f.write(r.content)
    f.close()

rpms = get_rpms ( puppetlab )

w = []

for i in rpms:
    print(i)
    g = gevent.spawn(download,i,"puppetlab",)
    w.append(g)

gevent.joinall(w)
gevent.sleep(0.3)

以上内容由PHP站长网【52php.cn】收集整理供大家参考研究

如果以上内容对您有帮助，欢迎收藏、点赞、推荐、分享。

（编辑：李大同）

【声明】本站内容均来自网络，其相关言论仅代表作者个人观点，不代表本站立场。若无意侵犯到您的权利，请及时与联系站长删除相关内容!