如何查询地址

In [1]:

import json
import re
import chardet
import requests

In [2]:

def get_text_from_html(url):
    """
    获取网页内容，匹配有地址的文本
    :param url: 获取地址的url
    :return: 返回包含地址的网页内容
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    content = resp.text

    prefix_ip = 'var ip_result = '
    for row in re.split('\\r\\n\\t', content):
        if prefix_ip in row and row:
            return row

In [3]:

ip = '180.97.125.228'
url = 'https://www.ip138.com/iplookup.asp?ip=%s&action=1' % ip
row = get_text_from_html(url)
row

Out [3]:

'\\t\\tvar ip_result = {"ASN?é????":"?\\xad??????????  ???? ", "iP??":"180.97.107.0 - 180.97.135.255", "????IPv6???·":"::B461:7DE4", "????IPv6???·":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"???ú", "prov":"?\\xad????", "city":"??????", "area":"", "idc":"", "yunyin":"????", "net":""}], "zg":1};'

In [4]:

# 这个网页文字的编码是unicode形式的gbk编码内容
# Unicode codepoints U+0000 to U+00FF和latin-1一一对应，编码后保留了等价的字节流数据。
row_byte = row.encode("latin-1")
row_byte

Out [4]:

b'\\t\\tvar ip_result = {"ASN\\xb9\\xe9\\xca\\xf4\\xb5\\xd8":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1\\xc4\\xcf\\xbe\\xa9\\xca\\xd0  \\xb5\\xe7\\xd0\\xc5 ", "iP\\xb6\\xce":"180.97.107.0 - 180.97.135.255", "\\xbc\\xe6\\xc8\\xddIPv6\\xb5\\xd8\\xd6\\xb7":"::B461:7DE4", "\\xd3\\xb3\\xc9\\xe4IPv6\\xb5\\xd8\\xd6\\xb7":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"\\xd6\\xd0\\xb9\\xfa", "prov":"\\xbd\\xad\\xcb\\xd5\\xca\\xa1", "city":"\\xc4\\xcf\\xbe\\xa9\\xca\\xd0", "area":"", "idc":"", "yunyin":"\\xb5\\xe7\\xd0\\xc5", "net":""}], "zg":1};'

In [5]:

# 通过字节码获取当前的编码为gb2312
encoding_detect = chardet.detect(row_byte)
encoding_detect

Out [5]:

{'encoding': 'GB2312', 'confidence': 0.99, 'language': 'Chinese'}

In [6]:

# 因此对其进行解码即可还原最初的unicode字符。
row_target = row_byte.decode("gb2312")
row_target

Out [6]:

'\\t\\tvar ip_result = {"ASN归属地":"江苏省南京市  电信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中国", "prov":"江苏省", "city":"南京市", "area":"", "idc":"", "yunyin":"电信", "net":""}], "zg":1};'

In [7]:

# 截取文本，转化为规范的json字符串
json_str = row_target.strip().replace('var ip_result = ', '').replace(';', '')
json_str

Out [7]:

'{"ASN归属地":"江苏省南京市  电信 ", "iP段":"180.97.107.0 - 180.97.135.255", "兼容IPv6地址":"::B461:7DE4", "映射IPv6地址":"::FFFF:B461:7DE4", "ip_c_list":[{"begin":3026283264, "end":3026290687, "ct":"中国", "prov":"江苏省", "city":"南京市", "area":"", "idc":"", "yunyin":"电信", "net":""}], "zg":1}'

In [8]:

# 转化为python里面的数据格式：字典
location_dict = json.loads(json_str)
location_dict

Out [8]:

{'ASN归属地': '江苏省南京市  电信 ',
 'iP段': '180.97.107.0 - 180.97.135.255',
 '兼容IPv6地址': '::B461:7DE4',
 '映射IPv6地址': '::FFFF:B461:7DE4',
 'ip_c_list': [{'begin': 3026283264,
   'end': 3026290687,
   'ct': '中国',
   'prov': '江苏省',
   'city': '南京市',
   'area': '',
   'idc': '',
   'yunyin': '电信',
   'net': ''}],
 'zg': 1}

In [9]:

# 获取中文地址
location = location_dict.get('ASN归属地', '').strip()
location

Out [9]:

'江苏省南京市  电信'