爬取騰訊招聘網信息

import requests
from bs4 import BeautifulSoup
from math import ceil

header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}


# 獲取崗位頁數
def getJobPage(url):
    ret = requests.get(url, headers=header)
    ret.encoding = "utf-8"  # 解決亂碼問題
    html = ret.text
    soup = BeautifulSoup(html, 'html.parser')
    # 獲取崗位總數，< span class ="lightblue total" > 512 < / span >
    totalJob = soup.select('span[class="lightblue total"]')[0].text
    jobPage = ceil(int(totalJob) / 10)
    return jobPage


def getJobOrder(url):
    ret = requests.get(url, headers=header)
    ret.encoding = "utf-8"  # 解決亂碼問題
    html = ret.text
    soup = BeautifulSoup(html, 'html.parser')
    # 工作職責
    jobRequests = soup.select('ul[class="squareli"]')[0].text
    # 工作要求
    jobOrder = soup.select('ul[class="squareli"]')[1].text
    return jobRequests, jobOrder


# 獲取崗位信息
def getJobInfo(url):
    myfile = open("tencent_job.txt", "a", encoding='gb18030', errors='ignore')  # 解決亂碼問題
    ret = requests.get(url, headers=header)
    ret.encoding = "utf-8"  # 解決亂碼問題
    html = ret.text
    soup = BeautifulSoup(html, 'html.parser')
    jobList = soup.find_all('tr', class_=['even', 'odd'])
    for job in jobList:
        # url
        jobUrl = "https://hr.tencent.com/" + job.select('td:nth-of-type(1) > a')[0]['href']
        # 職位名稱
        jobName = job.select('td:nth-of-type(1) > a')[0].text
        # 人數
        jobPeople = job.select('td:nth-of-type(3)')[0].text
        # 地點
        jobAddre = job.select('td:nth-of-type(4)')[0].text
        # 發佈時間
        jobTime = job.select('td:nth-of-type(5)')[0].text
        # 工作職責
        jobRequests = getJobOrder(jobUrl)[0]
        # 工作要求
        jobOrder = getJobOrder(jobUrl)[1]

        #print(jobName, jobUrl, jobAddre, jobPeople, jobTime, jobRequests, jobOrder)

        tt = jobName + " " + jobUrl + " " + jobAddre + " " + jobPeople + " " + jobTime + " " + jobRequests + " " + jobOrder
        myfile.write(tt + "\n")


if __name__ == '__main__':
    mainurl = 'https://hr.tencent.com/position.php?keywords=python'
    jobPage = getJobPage(mainurl)
    print(jobPage)
    for page in range(jobPage):
        pageUrl = 'https://hr.tencent.com/position.php?keywords=python&start=' + str(page * 10) + '#a'
        print("第" + str(page + 1) + "頁")
        getJobInfo(pageUrl)

# -*- coding:utf-8 -*-

import requests, json, time
from bs4 import BeautifulSoup


class tencent_hr(object):
    def __init__(self):
        self.base_url = "http://hr.tencent.com/position.php?"
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
        self.item_list = []
        self.page = 0

    # 發送請求
    def send_request(self, url, params={}):
        time.sleep(2)
        try:
            response = requests.get(url, params=params, headers=self.headers)
            return response.content
        except Exception as e:
            print e

    # 解析數據
    def parse_data(self, data):
        # 初始化
        bs = BeautifulSoup(data, 'lxml')

        # 獲取標簽-結果為列表
        data_list = bs.select('.even, .odd')

        # 將結果中的每一行數據提取出來
        for data in data_list:
            data_dict = {}
            data_dict['work_name'] = data.select('td a')[0].get_text()
            data_dict['work_type'] = data.select('td')[1].get_text()
            data_dict['work_count'] = data.select('td')[2].get_text()
            data_dict['work_place'] = data.select('td')[3].get_text()
            data_dict['work_time'] = data.select('td')[4].get_text()

            # 將每條字典數據添加進列表
            self.item_list.append(data_dict)

        # 判斷是否是最後一頁，條件：是否有noactive值
        # 先找到下一頁的標簽
        next_label = bs.select('#next')
        # 根據標簽獲取屬性class的值-返回結果為列表
        judge = next_label[0].get('class')

        return judge

    # 寫入文件
    def write_file(self):
        # 將列表轉換成字元串
        data_str = json.dumps(self.item_list)

        with open('04tencent_hr.json', 'w') as f:
            f.write(data_str)

    # 調度運行
    def run(self):
        while True:
            # 拼接參數
            params = {
                "keywords": "python",
                "tid": "0",
                "lid": "2156",
                "start": self.page,
            }

            # 發送請求
            data = self.send_request(self.base_url, params=params)

            # 解析數據
            judge = self.parse_data(data)

            self.page += 10
            print self.page

            # 如果到了最後一頁，出現noactive，跳出迴圈
            if judge:
                break

        self.write_file()


if __name__ == '__main__':
    spider = tencent_hr()
    spider.run()