Python解析CSDN网页文字内容

本文介绍如何使用Python对CSDN网站内容进行解析。

一、安装包

• beautifulsoup4
• html2text

二、实现功能

输入文章链接，输出文章的Markdown内容。

三、示例代码

# coding:utf-8

from bs4 import BeautifulSoup
import re
import html2text
import requests


def get_raw_html(url):
    if url is None:
        print("URL is None")
        return None
    user_angent = 'Mozilla/4.0 (compatibe; MSIE 5.5; Windows NT)'
    headers = {'User-Agent': user_angent}
    result = requests.get(url, headers=headers)
    if result.status_code == 200:
        result.encoding = 'utf-8'
        return result.text
    print("URL is invalid")
    return None


def get_text_by_url(url):
    with open('web.html', 'w', encoding='utf-8') as file_obj:
        file_obj.write(get_raw_html(url))
    soup = BeautifulSoup(open('web.html', encoding='utf-8'))
    raw_text = str(soup.article)
    if raw_text:
        result_text = clear_tab(raw_text)
        return result_text


def clear_tab(raw_text):
    re_cdata = re.compile(u'//<![CDATA[[^>]*//]]', re.I)
    re_script = re.compile(u'<s*script[^>]*>[^<]*<s*/s*scripts*>', re.I)
    re_style = re.compile(u'<s*style[^>]>[^<]*<s*/s*styles*>', re.I)
    re_br = re.compile(u'<brs*?/?>')
    re_h = re.compile(u'</?w+[^>]*>')
    re_comment = re.compile(u'<!--[^>]*-->')

    s = re_cdata.sub('', raw_text)
    s = re_script.sub('', s)
    s = re_style.sub('', s)
    s = re_br.sub('n', s)
    s = re_h.sub('', s)
    s = re_comment.sub('', s)
    blank_line = re.compile('n+')
    s = blank_line.sub('n', s)
    s = replaceCharEntity(s)
    with open('web.txt', 'w', encoding='utf-8') as file_obj:
        file_obj.write(s)
    return html2text.html2text(s)


def replaceCharEntity(s):
    CHAR_ENTITIES = {
        'nbsp': '',
        '160': '',
        'lt': '<',
        '60': '<',
        'gt': '>',
        '62': '>',
        'amp': '&',
        '38': '&',
        'quot': '"',
        '34': '"'
    }
    re_charEntity = re.compile(r'&#?(?P<name>w+);')
    sz = re_charEntity.search(s)
    while sz:
        entity = sz.group()
        key = sz.group('name')
        try:
            s = re_charEntity.sub(CHAR_ENTITIES[key], s, 1)
            sz = re_charEntity.search(s)
        except KeyError:
            s = re_charEntity.sub('', s, 1)
            sz = re_charEntity.search(s)
    return s


def main():
    url = input('请输入CSDN网站文章连接：')
    dst_file = 'output.md'
    result = get_text_by_url(url.strip())
    print(result)
    with open(dst_file, 'w', encoding='utf-8') as file_obj:
        file_obj.write(result + '\n')


if __name__ == "__main__":
    main()

赏

谢谢你请我吃糖果

微信