# coding = UTF-8
# 爬取自己编写的html链接中的PDF文档,网址：file:///E:/ZjuTH/Documents/pythonCode/pythontest.html

import urllib.request
import re
import os
from lxml import etree
import requests

url = 'http://jb.sznews.com/PC/layout/202105/20/node_A01.html'
response = requests.get(url)
response.encoding=response.apparent_encoding
print(response)
label=etree.HTML(response.text)
#提取这个页面中所有的标签信息
content=label.xpath('//div[@class="Therestlist"]/text()')
#提取span标签中class名为"is-text-small is-text-grey"的内容信息,并且存入一个列表中
# print(content[0])
#打印获得的文本信息
exit()

# open the url and read
def getHtml(url):
    page = urllib.request.urlopen(url)
    html = page.read()
    page.close()
    return html

###
def getTags(html):
 reg = r'<div class="Therestlist">([\s\S]+?)</div>'
 pattern= re.compile(reg)
 tags= re.findall(pattern, html)
 return tags

# compile the regular expressions and find
# all stuff we need
def getUrl(html):
    reg = r'([A-Z]\d+)' #匹配了G176200001
    url_re = re.compile(reg)
    url_lst = url_re.findall(html.decode('UTF-8')) #返回匹配的数组
    print (url_lst)
    exit()
    return(url_lst)

def getFile(url):
    file_name = url.split('/')[-1]
    u = urllib.request.urlopen(url)
    f = open(file_name, 'wb')

    block_sz = 8192
    while True:
        buffer = u.read(block_sz)
        if not buffer:
            break

        f.write(buffer)
    f.close()
    print ("Sucessful to download" + " " + file_name)


root_url = 'http://jb.sznews.com/attachment/pdf/202105/20/'  #下载地址中相同的部分

raw_url = 'http://jb.sznews.com/PC/layout/202105/20/node_A01.html'

html = getHtml(raw_url)
tags = getTags(html)
url_lst = getUrl(tags)

os.mkdir('pdf_download')
os.chdir(os.path.join(os.getcwd(), 'pdf_download'))

for url in url_lst[:]:
    url = root_url + url+'/'+url+'.pdf'  #形成完整的下载地址
    getFile(url)