一键获取西刺代,理IP

本文来源吾爱破解论坛

本帖最后由 null119 于 2018-12-28 15:41 编辑

使用Python爬虫避免不了会遇到IP限制的问题，索性写个一键获取代,理 IP，包含检查代,理 IP的可用性，供需要的同学参考


引用库：requests,lxml


获取代,理IP网站：西刺代,理

上图：

proxy.png (145.4 KB, 下载次数: 1)

下载附件  保存到相册

2018-12-28 15:30 上传

import requests
from lxml import etree

url = "http://2018.ip138.com/ic.asp"
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"}
dlurl = [
        'https://www.xicidaili.com/wt/',  # 国内http
        'https://www.xicidaili.com/wn/',  # 国内https
        'https://www.xicidaili.com/nt/',  # 国内普通
        'https://www.xicidaili.com/nn/'  # 国内高匿
    ]

#获取西刺代,理IP页面数据
def gethtml(url):
    porxy=[]
    r = requests.get(url,headers=headers)
    r.encoding='utf-8'
    ehtml = etree.HTML(r.text)
    ip  =ehtml.xpath('//*[@class="odd"]/td[2]/text()')
    port=ehtml.xpath('//*[@class="odd"]/td[3]/text()')
    j=0
    for i in ip:
        porxy.append(i+':'+port[j])
        j+=1
    ip  =ehtml.xpath('//*[@class=""]/td[2]/text()')
    port=ehtml.xpath('//*[@class=""]/td[3]/text()')
    j=0
    for i in ip:
        porxy.append(i+':'+port[j])
        j+=1
    return porxy

#检查代,理是否可用
def chkproxy(ipport,timeout):
    proxies = {"http": "http://" + ipport}
    try:
        response = requests.get(url, headers=headers, proxies=proxies, timeout=timeout)
        if response.status_code == 200:
            return True
    except OSError:
        pass

def exec(url,pagenum,timeout):
    f = open('C:/ProxyIP.txt', mode='w+')
    for i in range(1,pagenum+1):
        print('开始获取第'+str(i)+'页代,理IP数据...')
        porxy=gethtml(url+str(i))
        j=0
        print('开始检查第'+str(i)+'页代,理IP...')
        for i in porxy:
            if chkproxy(i,timeout)==True:
                f.write(i+'\n')
            j+=1
    print('任务完成,代,理IP保存在C:\Proxy.txt')

if __name__ == '__main__':
    #exec(代,理IP网页url,获取页数,超时设置)  每页是100条代,理IP数据
    exec(dlurl[0],3,0.3)