import requests
from pyquery import PyQuery
import re


def get_real(o_url):
    """获取重定向url指向的网址"""
    r = requests.get(o_url, allow_redirects=False)  # 禁止自动跳转
    if r.status_code == 302:
        try:
            return r.headers['location']  # 返回指向的地址
        except:
            pass
    return o_url  # 返回源地址


keyword = 'liuyukuan'
# keyword = '刘玉宽'
my_url = "https://www.baidu.com/s?wd=" + keyword

# 定制请求头
my_headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36"
}

# 获取网页文档,不加headers参数，百度返回值不正确
html = PyQuery(url=my_url, headers=my_headers, encoding="utf-8")

titles = []
urls = []
for item in html('#content_left  h3.t a').items():
    titles.append(item.text())
    urls.append(get_real(item.attr('href')))
print(titles)
print(urls)