Scrapy-从数据库取出IP并判断是否

发布时间:2019-05-30 21:09:49编辑:auto阅读(1869)

    import pymysql
    import requests
    
    conn = pymysql.connect(host="localhost",user="root",passwd="root",db="CDL",charset="utf8")
    cursor = conn.cursor()
    class IpUtils(object):
    
        # 删除无效ip
        def delete_ip(self,ip):
            sql ="delete from xiciproxy where ip='{0}'".format(ip)
            cursor.execute(sql)
            conn.commit()
            return True
    
        # 判断ip是否可用
        def judge_ip(self,type,ip,port):
            http_url ="http://www.baidu.com"
            proxy_url ="{0}://{1}:{2}".format(type,ip,port)
            print("url:"+ proxy_url)
            try:
                proxy_dict = {
                    type: proxy_url, # type 为https 或者http 数据库存储的
                }
                response = requests.get(http_url,proxies=proxy_dict)
            exceptExceptionase:
                self.delete_ip(ip)
                print("invalid ip and port")
                return False
                code = response.status_code
                if code >=200 and code <300:
                    print("effective ip")
                    return True
            else:
                print("invalid ip and port")
                self.delete_ip(ip)
                return False
        # 随机获取IP
        def get_random_ip(self):
            random_sql ="""
                SELECT type, ip, port FROM xiciproxy
                ORDER BY RAND()
                LIMIT 1
            """
            result = cursor.execute(random_sql)
            for ip_info in cursor.fetchall():
                type = ip_info[0]
                ip = ip_info[1]
                port = ip_info[2]
                judge_re =self.judge_ip(type,ip,port)
                ifjudge_re:
                return"{0}://{1}:{2}".format(type,ip,port)
            else:
                return self.get_random_ip()
    if__name__ =="__main__":
        Ip = IpUtils()
        print(Ip.get_random_ip())

    2020考研打卡第二十天,星辰之变,骄阳岂是终点。

    人生不如意十之八九,我怎么能放弃呢,老子一定要做大做强!!!

关键字