#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests, bs4

def url2soup(url, headers={}, data=None):
    """url -> soup
    
    处理任意编码的网页，返回BeautifulSoup对象
    """

    if data:
        response = requests.post(url, data=data, headers=headers)
    else:
        response = requests.get(url, headers=headers)

    encodings = requests.utils.get_encodings_from_content(response.text)
    if encodings:
        encoding = encodings[0]
    else:
        encoding = response.apparent_encoding
    encode_content = response.content.decode(encoding, 'replace')
    return bs4.BeautifulSoup(encode_content, "lxml")