【作业】2022.5.16 使用selenium获取数据并写入csv文件

作业：使用selenium获得数据并整理到csv表格中

import csv
import os
import re
from tqdm import tqdm
from bs4 import BeautifulSoup
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.common.keys import Keys
from time import sleep
from random import random


def options_change():
    options = ChromeOptions()
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
    return options


def create_file(path, head):
    if not os.path.exists(path):
        writer = csv.writer(open(path, 'a', encoding='utf-8', newline=''))
        writer.writerow(head)
    else:
        writer = csv.writer(open(path, 'a', encoding='utf-8', newline=''))
    return writer


def search_target(box, target):
    box.send_keys(target)
    box.send_keys(Keys.ENTER)


def get_all(page, selector):
    soup = BeautifulSoup(page.page_source, 'lxml')
    return soup.select(selector)


def single_house_info(info):
    # 获取二手房名字
    house_name = info.select_one('div.title > a').text

    # 获取二手房地址
    position = info.select_one('div.flood > div').text
    position = re.sub(r's', '', position)

    # 获取二手房具体信息
    house_details = info.select_one('div.address > div').text.split('|')

    # 获取二手房面积
    house_area = house_details[1]

    # 获取建造时间
    if len(house_details) == 6:
        building_time = '暂无'
    else:
        building_time = house_details[-2]

    # 获取二手房总价
    total_price = info.select_one('div.totalPrice.totalPrice2').text

    # 获取二手房每平方米单价
    unit_price = info.select_one('div.unitPrice > span').text

    return [house_name, position, house_area, building_time, total_price, unit_price]


if __name__ == '__main__':
    writer = create_file('files/houses.csv', ['二手房名', '地址', '面积', '建造时间', '总价', '单价'])

    # 打开主页
    options = options_change()
    house = Chrome(options=options)
    house.get('https://cd.lianjia.com/')
    sleep(random() * 2 + 1)

    # 搜索目标
    search_box = house.find_element_by_id('keyword-box')
    search_target(search_box, '高新')
    sleep(random() * 2 + 1)

    # 开始按页数循环（100页）
    for _ in tqdm(range(99)):
        # 得到本页所有的二手房信息
        all_infos = get_all(house, 'ul.sellListContent > li')

        # 得到单独每个二手房的信息
        for single_info in tqdm(all_infos):
            writer.writerow(single_house_info(single_info))

        # 用空行分隔
        writer.writerow([])

        # 下一页
        next = house.find_element_by_xpath('//*[@id="content"]/div[1]/div[7]/div[2]/div/a[last()]')
        house.execute_script('window.scrollBy(0, 8000)')
        next.click()
        sleep(random() * 2 + 1)

    house.close()