添加匿名度检查, 扩展/get

api/proxyApi.py

@@ -42,7 +42,7 @@ def force_type(cls, response, environ=None):
 app.response_class = JsonResponse
 
 api_list = [
-    {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"},
+    {"url": "/get", "params": "type: 'https'|'', region: '代理地区, 中文', anonymous: '任意: -1, '", "desc": "get a proxy"},
     {"url": "/pop", "params": "", "desc": "get and delete a proxy"},
     {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"},
     {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"},
@@ -59,7 +59,12 @@ def index():
 @app.route('/get/')
 def get():
     https = request.args.get("type", "").lower() == 'https'
-    proxy = proxy_handler.get(https)
+    region = request.args.get("region", "").lower()
+    try:
+        anonymous = int(request.args.get("anonymous", '').lower())
+    except:
+        anonymous = -1
+    proxy = proxy_handler.get(https, region, anonymous)
     return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 

db/dbClient.py

@@ -86,8 +86,8 @@ def __initDbClient(self):
                                                                                      password=self.db_pwd,
                                                                                      db=self.db_name)
 
-    def get(self, https, **kwargs):
-        return self.client.get(https, **kwargs)
+    def get(self, https, region, anonymous, **kwargs):
+        return self.client.get(https, region, anonymous, **kwargs)
 
     def put(self, key, **kwargs):
         return self.client.put(key, **kwargs)

db/redisClient.py

@@ -47,19 +47,35 @@ def __init__(self, **kwargs):
                                                                    socket_timeout=5,
                                                                    **kwargs))
 
-    def get(self, https):
+    def get(self, https, region, anonymous):
         """
         返回一个代理
         :return:
         """
+        items = self.__conn.hvals(self.name)
+        proxies = list(items)
         if https:
-            items = self.__conn.hvals(self.name)
             proxies = list(filter(lambda x: json.loads(x).get("https"), items))
-            return choice(proxies) if proxies else None
-        else:
-            proxies = self.__conn.hkeys(self.name)
-            proxy = choice(proxies) if proxies else None
-            return self.__conn.hget(self.name, proxy) if proxy else None
+        if region != '':
+            proxies = list(filter(lambda x: region in json.loads(x).get("region"), iter(proxies)))
+        if anonymous != -1:
+            proxies = list(filter(lambda x: json.loads(x).get("anonymous") == anonymous, iter(proxies)))
+        return choice(proxies) if proxies else None
+
+        # if https:
+        #     items = self.__conn.hvals(self.name)
+        #     proxies = list(filter(lambda x: json.loads(x).get("https"), items))
+        #     if region != '':
+        #         proxies = list(filter(lambda x: region in json.loads(x).get("region"), iter(proxies)))
+        #     return choice(proxies) if proxies else None
+        # elif region != '':
+        #     items = self.__conn.hvals(self.name)
+        #     proxies = list(filter(lambda x: region in json.loads(x).get("region"), items))
+        #     return choice(proxies) if proxies else None
+        # else:
+        #     proxies = self.__conn.hkeys(self.name)
+        #     proxy = choice(proxies) if proxies else None
+        #     return self.__conn.hget(self.name, proxy) if proxy else None
 
     def put(self, proxy_obj):
         """

db/ssdbClient.py

@@ -47,19 +47,20 @@ def __init__(self, **kwargs):
                                                                    socket_timeout=5,
                                                                    **kwargs))
 
-    def get(self, https):
+    def get(self, https, region, anonymous):
         """
         从hash中随机返回一个代理
         :return:
         """
+        items = self.__conn.hgetall(self.name).values()
+        proxies = list(items)
         if https:
-            items_dict = self.__conn.hgetall(self.name)
-            proxies = list(filter(lambda x: json.loads(x).get("https"), items_dict.values()))
-            return choice(proxies) if proxies else None
-        else:
-            proxies = self.__conn.hkeys(self.name)
-            proxy = choice(proxies) if proxies else None
-            return self.__conn.hget(self.name, proxy) if proxy else None
+            proxies = list(filter(lambda x: json.loads(x).get("https"), items))
+        if region != '':
+            proxies = list(filter(lambda x: region in json.loads(x).get("region"), iter(proxies)))
+        if anonymous != -1:
+            proxies = list(filter(lambda x: json.loads(x).get("anonymous") == anonymous, iter(proxies)))
+        return choice(proxies) if proxies else None
 
     def put(self, proxy_obj):
         """

handler/proxyHandler.py

@@ -26,14 +26,16 @@ def __init__(self):
         self.db = DbClient(self.conf.dbConn)
         self.db.changeTable(self.conf.tableName)
 
-    def get(self, https=False):
+    def get(self, https=False, region='', anonymous=-1):
         """
         return a proxy
         Args:
             https: True/False
+            region:
+            anonymous:
         Returns:
         """
-        proxy = self.db.get(https)
+        proxy = self.db.get(https, region, anonymous)
         return Proxy.createFromJson(proxy) if proxy else None
 
     def pop(self, https):

helper/check.py

@@ -13,6 +13,9 @@
 """
 __author__ = 'JHao'
 
+import requests
+import json
+
 from util.six import Empty
 from threading import Thread
 from datetime import datetime
@@ -40,6 +43,8 @@ def validator(cls, proxy):
         proxy.check_count += 1
         proxy.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         proxy.last_status = True if http_r else False
+        proxy._region = cls.region_get(proxy.proxy)
+        proxy._anonymous = cls.anonymousValidator(proxy.proxy, proxy.https)
         if http_r:
             if proxy.fail_count > 0:
                 proxy.fail_count -= 1
@@ -69,6 +74,35 @@ def preValidator(cls, proxy):
                 return False
         return True
 
+    @classmethod
+    def region_get(cls, proxy):
+        try:
+            r = requests.get(url='https://searchplugin.csdn.net/api/v1/ip/get', params={'ip': proxy.split(':')[0]})
+            return json.loads(r.text)['data']['address']
+        except:
+            return '未知或请求失败'
+
+    @classmethod
+    def anonymousValidator(cls, proxy, https):
+        if https:
+            url = 'https://httpbin.org/get'
+            proxy = {'https': proxy}
+        else:
+            url = 'http://httpbin.org/get'
+            proxy = {'http': proxy}
+
+        r = requests.get(url, proxies=proxy)
+        r = json.loads(r.text)
+        try:
+            if ',' in r.get('origin'):
+                return 0
+            elif r.get('headers').get('Proxy-connecttion', False):
+                return 1
+            else:
+                return 2
+        except:
+            return -1
+
 
 class _ThreadChecker(Thread):
     """ 多线程检测 """