• Home
  • Categories
  • WIKI
  • LANGUAGE: en
    • 1750款商业游戏的小爬虫
    Theme area 1110 views · 1 replies ·
    Tofloor
    poster avatar
    shouhuanxiaoji
    deepin
    2018-05-16 01:56
    Author
    本帖最后由 shouhuanxiaoji 于 2018-5-16 13:11 编辑

    学了几天python3，手痒写了一段爬虫代码，爬的rutracker上的linux游戏及下载链接
    一共1750款，也算满足几年前的心愿。
    会不断完善，加上其他网站上的游戏，预计能爬上万款吧，回头有空慢慢测试打包appimage
    对版权有洁癖的请绕行。
    代码比较烂，欢迎提建议。
    请先sudo pip3 install pyquery
    1. #!/usr/bin/env python3
    2. from pyquery import PyQuery as pq
    3. import urllib.parse
    4. import urllib.request
    5. import os,time

    7. URLFIRST = "https://rutracker.org/forum/viewforum.php?f=1992"
    8. currentpath = os.path.abspath('.')
    9. listpagepath = os.path.join(currentpath, 'list')
    10. outputpath = os.path.join(currentpath, 'rutracker-output')
    11. outputfile = os.path.join(outputpath, 'output.txt')
    12. gamelist = {}
    13. if not os.path.exists(listpagepath):
    14.     os.mkdir(listpagepath)


    17. def CachePage(IndexPageNum=1):
    18.     urldata = {}
    19.     # 首页无此参数，第二页为50，第三页为100，依次类推
    20.     urldata["start"] = IndexPageNum * 50
    21.     urlvalue = urllib.parse.urlencode(urldata)
    22.     if (IndexPageNum == 0):
    23.         url = URLFIRST
    24.     else:
    25.         url = URLFIRST + "&" + urlvalue
    26.     response = urllib.request.urlopen(url)
    27.     htmlcontent = response.read().decode("Windows-1251")
    28.     controledfile = open(os.path.join(listpagepath, str(IndexPageNum + 1) + ".html"), "w", encoding="utf-8")
    29.     controledfile.write(htmlcontent)
    30.     controledfile.close()
    31.     response.close()
    32.     print("第" + str(IndexPageNum + 1) + "页已获取完毕！")


    35. def CacheMagnet():
    36.     htmldom = pq(URLFIRST)
    37.     indexnum = htmldom('a.pg').eq(-2).text()
    38.     indexnum = int(indexnum)
    39.     if not os.path.exists(outputpath):
    40.         os.mkdir(outputpath)
    41.     if not os.path.exists(outputfile):
    42.         op = open(outputfile, 'w', encoding='utf-8')
    43.         op.close()
    44.     if (len([i for i in os.listdir(listpagepath) if os.path.isfile(os.path.join(listpagepath, i))]) != indexnum):
    45.         for i in range(0, indexnum):
    46.             CachePage(i)
    47.     for i in os.listdir(listpagepath):
    48.         if os.path.isfile(os.path.join(listpagepath, i)):
    49.             htmlfile = open(os.path.join(listpagepath, i), 'r', encoding='utf-8', errors='ignore')
    50.             htmldata = pq(htmlfile.read())
    51.             gamelistnum = len(htmldata('a.tt-text'))
    52.             for m in range(0, gamelistnum):
    53.                 innerhref = htmldata('a.tt-text').eq(m).attr('href')
    54.                 innerurl = 'https://rutracker.org/forum/' + innerhref
    55.                 innerdom = pq(url = innerurl)
    56.                 op = open(outputfile, 'a', encoding='utf-8')
    57.                 gamelist['size'] = htmldata('a.dl-stub').eq(m).text()
    58.                 gamelist['name'] = innerdom('div.post_body').children('span').eq(0).text()
    59.                 gamelist['magnet'] = innerdom('a.magnet-link').attr('href')
    60.                 op.write(str(gamelist))
    61.                 op.close
    62.                 time.sleep(2)
    63.             htmlfile.close()


    66. CacheMagnet()
    Copy the Code




    Reply Favorite View the author
    All Replies
    avatar
    shouhuanxiaoji
    deepin
    2018-05-16 01:57
    #1
    应该还有个ssl的报错，百度一下，缺个库，pip装一下。
    Reply View the author