添加hifini_music爬虫代码

hifini_music/index.js

@@ -0,0 +1,159 @@
+const fs = require('fs');
+const { getApiResult } = require('../utils/requestUtils');
+const dbUtils = require("../utils/dbPoolUtils");
+const sleepUtils = require("../utils/sleepUtils");
+
+// 数据库连接池
+dbUtils.create({
+    database: "neteasemusic",   // 指定数据库
+    connectionLimit: 10,        // 设置数据库连接池数量
+});
+global.dbUtils = dbUtils;
+
+const dataManager = require('./src/dataManager');
+const requestUtils = require('../utils/requestUtils');
+
+async function main() {
+    async function timeout1() {
+        await getList();
+        setTimeout(timeout1, 2000);
+    }
+    timeout1();
+
+    async function timeout2() {
+        await startFetchDetail();
+        setTimeout(timeout2, 2000);
+    }
+    timeout2();
+
+    async function timeout3() {
+        await startFetchRealUrl();
+        setTimeout(timeout3, 2000);
+    }
+    timeout3();
+}
+
+// 爬取列表页，获得歌曲详情页
+async function getList() {
+
+    let forumId = 12; // 分类id
+    let beginPage = 125; // 起始页
+    let endPage = 165; // 结束页
+    for (let page = beginPage; page <= endPage; page++) {
+        let url = `https://hifini.com/forum-${forumId}-${page}.htm?orderby=tid`; // 按照发帖时间排序
+        console.log(`getList   \t| ${beginPage}/${page}/${endPage} | forumId: ${forumId} | ${url}`);
+
+        // let html = fs.readFileSync("./1.html", "utf8");
+        let html = await getApiResult(url);
+        // fs.writeFileSync("./1.html", html);
+
+        var matcher = html.matchAll(/<a href="thread-(\d{1,15}).htm">(.*?)<\/a>/g);
+        var m = matcher.next();
+        var threadList = [];
+        while (!m.done) {
+            if (!/^.*?\[[-\/\.A-Za-z0-9]+?\]$/.exec(m.value[2])) {
+                console.log(`跳过 ${m.value[2]}`);
+            } else {
+                threadList.push({
+                    forum_id: forumId,
+                    thread_id: Number(m.value[1]),
+                    title: m.value[2]
+                });
+            }
+            m = matcher.next();
+        }
+        await dataManager.thread.insertCollection(threadList);
+        await sleepUtils.sleep(1000);
+    }
+}
+
+async function startFetchDetail() {
+    let idsToFetch = await dataManager.thread.getIdsToFetch();
+    idsToFetch = idsToFetch.map(item => item.thread_id);
+    // console.log(idsToFetch);
+    for (let i = 0; i < idsToFetch.length; i++) {
+        const threadId = idsToFetch[i];
+        console.log(`getDetail\t| ${i + 1}/${idsToFetch.length} | threadId: ${threadId}`);
+        await getDetail(threadId);
+        await sleepUtils.sleep(1000);
+    }
+}
+
+async function getDetail(threadId) {
+
+    let url = `https://hifini.com/thread-${threadId}.htm`;
+
+    // let html = fs.readFileSync("./1.html", "utf8");
+    let html = await getApiResult(url);
+    // fs.writeFileSync("./1.html", html);
+
+    // 解析到音乐信息
+    var matcher = /var ap4 = new APlayer\(([\S\s]*?)\);/.exec(html);
+    if (!matcher) {
+        await dataManager.thread.update(threadId, { music_title: "未解析到音乐" });
+        console.log("未解析到音乐，跳过");
+        return;
+    }
+    try {
+        let arrStr = matcher[1];
+        // console.log(arrStr);
+        eval(`let document = { getElementById: () => {} }; var arr = ${arrStr};`);
+        var music = arr.music[0];
+        // console.log(music);
+    } catch (e) {
+        console.error("解析失败", e);
+        return;
+    }
+
+    var matcher = html.matchAll(/<a href='tag-(\d{1,15}).htm'><i class="icon-tag"><\/i>(.*?)<\/a>/g);
+    var m = matcher.next();
+    var tagList = [];
+    while (!m.done) {
+        tagList.push({
+            tag_id: Number(m.value[1]),
+            tag_name: m.value[2]
+        });
+        m = matcher.next();
+    }
+
+    await dataManager.tag.insertCollection(tagList);
+
+    await dataManager.thread_tag.insertCollection(tagList.map(tag => {
+        return {
+            thread_id: threadId,
+            tag_id: tag.tag_id
+        };
+    }));
+
+    await dataManager.thread.update(threadId, {
+        music_title: music.title,
+        music_author: music.author || "",
+        music_url: music.url,
+        music_pic: music.pic || ""
+    });
+    // console.log("done");
+}
+
+async function startFetchRealUrl() {
+    let urlsToFetch = await dataManager.thread.getIdsToFetchRealUrl();
+    // console.log(urlsToFetch.map(item => item.thread_id));
+    urlsToFetch = urlsToFetch.map(item => { return { threadId: item.thread_id, fakeUrl: item.music_url } });
+    for (let i = 0; i < urlsToFetch.length; i++) {
+        const urlToFetch = urlsToFetch[i];
+        console.log(`getRealUrl\t| ${i + 1}/${urlsToFetch.length} | threadId: ${urlToFetch.threadId} | ${urlToFetch.fakeUrl}`);
+        await getRealUrl(urlToFetch);
+        await sleepUtils.sleep(1000);
+    }
+}
+
+async function getRealUrl(urlToFetch) {
+    let { threadId, fakeUrl } = urlToFetch;
+    try {
+        let url = await requestUtils.getRedirectUrl(`https://hifini.com/${fakeUrl}`);
+        result = await dataManager.thread.update(threadId, { music_real_url: url });
+    } catch (e) {
+        console.log("重定向地址获取失败");
+    }
+}
+
+main();

hifini_music/src/dataManager.js

@@ -0,0 +1,43 @@
+const dbUtils = global.dbUtils;
+
+let insertCollectionTemplate = async (tableName, dataList) => {
+    if (dataList.length == 0) return;
+    return await dbUtils.query(`
+        INSERT INTO ${tableName} ( ${Object.keys(dataList[0]).map(field => `\`${field}\``).join(",")} ) VALUES ?
+        ON DUPLICATE KEY UPDATE ${Object.keys(dataList[0]).map(field => `${field}=VALUES(${field})`).join(", ")}
+    `, [dataList.map(item => Object.values(item))]);
+}
+
+module.exports = {
+
+    thread: {
+        insertCollection: async (threadList) => {
+            return await insertCollectionTemplate("hifini_thread", threadList);
+        },
+
+        update: async (threadId, threadInfo) => {
+            return await dbUtils.query(`UPDATE hifini_thread SET ? WHERE thread_id = ${threadId}`, threadInfo);
+        },
+
+        getIdsToFetch: async () => {
+            return await dbUtils.query(`SELECT thread_id FROM hifini_thread where music_title='' and music_pic='' and music_url=''`);
+        },
+
+        getIdsToFetchRealUrl: async () => {
+            return await dbUtils.query(`SELECT thread_id,music_url FROM hifini_thread where music_url like 'get_music.php?key=%' and music_real_url=''`);
+        }
+    },
+
+    tag: {
+        insertCollection: async (tagList) => {
+            return await insertCollectionTemplate("hifini_tag", tagList);
+        },
+    },
+
+    thread_tag: {
+        insertCollection: async (tagList) => {
+            return await insertCollectionTemplate("hifini_thread_tag_relation", tagList);
+        },
+    },
+
+};

utils/requestUtils.js

@@ -35,8 +35,24 @@ async function query(opts) {
     return return_data;
 }
 
+async function getRedirectUrl(url) {
+    return await new Promise((resolve, reject) => {
+        request({
+            url: url,
+            followRedirect: false
+        }, function (err, res, body) {
+            if (err) {
+                reject(err);
+            }
+            // console.log(res.headers.location);
+            resolve(res.headers.location);
+        });
+    });
+}
+
 module.exports = {
     get: get,
     getApiResult: getApiResult,
     query: query,
+    getRedirectUrl: getRedirectUrl,
 }