openhub/python at master · SecReXus/openhub · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#-*- coding:utf-8 -*-


import bs4
from bs4 import BeautifulSoup
import re
import requests
import os
import traceback
'''
获得页面信息
'''
def getHtmlText(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

'''
获得content
'''
def getImgContent(url):
    head = {"user-agent":"Mozilla/5.0"}
    try:
        r = requests.get(url,headers=head,timeout=30)
        print('status_code:'+r.status_code)
        r.raise_for_status()
        return r.content
    except:
        return None
'''
获得页面中的表情的链接
'''
def getTypeUrlList(html,typeUrlList):
    soup = BeautifulSoup(html,'html.parser')
    divs = soup.find_all("div",attrs={"class":"up"})
    for div in divs:
        a = div.find("div",attrs={"class":"num_1"}).find("a")
        title = a.attrs["title"]
        typeUrl = a.attrs["href"]
        typeUrlList.append((title,typeUrl))
def getImgUrlList(typeUrlList,imgUrlDict):
    for tuple in typeUrlList:
        title = tuple[0]
        url = tuple[1]
        title_imgUrlList=[]
        html = getHtmlText(url)
        soup = BeautifulSoup(html,"html.parser")
        # print(soup.prettify())

        div = soup.find("div",attrs={"class":"img_text"})
        # print(type(div))
        imgDiv = div.next_sibling.next_sibling
        # print(type(imgDiv))
        imgs = imgDiv.find_all("img")
        for img in imgs:
            src = img.attrs["src"]
            title_imgUrlList.append(src)
        imgUrlDict[title] = title_imgUrlList
def getImage(imgUrlDict,file_path):
    head = {"user-agent":"Mizilla/5.0"}
    countdir = 0
    for title,imgUrlList in imgUrlDict.items():
        # print(title+":"+str(imgUrlList))
        try:
            dir = file_path+title
            if not os.path.exists(dir):
                os.mkdir(dir)
            countfile =0
            for imgUrl in imgUrlList:
                path = dir + "/" + imgUrl.split("/")[-1]
                # print(path)
                # print(imgUrl)
                if not os.path.exists(path):
                    r = requests.get(imgUrl,headers=head,timeout=30)
                    r.raise_for_status()
                    with open(path,"wb") as f:
                        f.write(r.content)
                        f.close()
                        countfile = countfile+1
                        print("当前进度文件夹进度{:.2f}%".format(countfile*100/len(imgUrlList)))
            countdir =countdir +1
            print("文件夹进度{:.2f}%".format(countdir*100/len(imgUrlDict)))

        except:
            print(traceback.print_exc())

def main():
    pages = 30
    root = "http://sc.chinaz.com/biaoqing/"
    url = "http://sc.chinaz.com/biaoqing/index.html"
    file_path = "E://biaoqing1/"
    imgUrlDict ={}
    typeUrlList = []
    html = getHtmlText(url)
    getTypeUrlList(html,typeUrlList)
    getImgUrlList(typeUrlList,imgUrlDict)
    getImage(imgUrlDict,file_path)
    for page in range(pages):
        url = root + "index_" +str(page)+".html"
        imgUrlDict = {}
        typeUrlList = []
        html = getHtmlText(url)
        getTypeUrlList(html,typeUrlList)
        getImgUrlList(typeUrlList,imgUrlDict)
        getImage(imgUrlDict,file_path)

main()