coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

添加SQL建表语句；考虑音乐页面多收首歌的情况；Bugfix

Browse Source
This commit is contained in:
小墨
2022-10-28 01:14:44 +08:00
parent 6ce6b0cd46
commit ddde1b28f9
3 changed files with 98 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
hifini_music/index.js
View File

@@ -14,11 +14,11 @@ const dataManager = require('./src/dataManager');
const requestUtils = require('../utils/requestUtils');
async function main() {
async function timeout1() {
await getList();
setTimeout(timeout1, 2000);
}
timeout1();
// async function timeout1() {
// await getList();
// setTimeout(() => console.log("getList已完成"), 2000);
// }
// timeout1();
async function timeout2() {
await startFetchDetail();
@@ -36,9 +36,9 @@ async function main() {
// 爬取列表页，获得歌曲详情页
async function getList() {
let forumId = 12; // 分类id
let beginPage = 125; // 起始页
let endPage = 165; // 结束页
let forumId = 1; // 分类id
let beginPage = 1; // 起始页
let endPage = 23; // 结束页
for (let page = beginPage; page <= endPage; page++) {
let url = `https://hifini.com/forum-${forumId}-${page}.htm?orderby=tid`; // 按照发帖时间排序
console.log(`getList \t| ${beginPage}/${page}/${endPage} | forumId: ${forumId} | ${url}`);
@@ -51,15 +51,15 @@ async function getList() {
var m = matcher.next();
var threadList = [];
while (!m.done) {
if (!/^.*?\[[-\/\.A-Za-z0-9]+?\]$/.exec(m.value[2])) {
console.log(`跳过 ${m.value[2]}`);
} else {
threadList.push({
forum_id: forumId,
thread_id: Number(m.value[1]),
title: m.value[2]
});
}
// if (!/^.*?\[[-\/\.A-Za-z0-9]+?\]$/.exec(m.value[2])) {
// console.log(`跳过 ${m.value[2]}`);
// } else {
threadList.push({
forum_id: forumId,
thread_id: Number(m.value[1]),
title: m.value[2]
});
// }
m = matcher.next();
}
await dataManager.thread.insertCollection(threadList);
@@ -90,7 +90,7 @@ async function getDetail(threadId) {
// 解析到音乐信息
var matcher = /var ap4 = new APlayer\(([\S\s]*?)\);/.exec(html);
if (!matcher) {
await dataManager.thread.update(threadId, { music_title: "未解析到音乐" });
await dataManager.thread.update(threadId, 0, { music_title: "未解析到音乐" });
console.log("未解析到音乐，跳过");
return;
}
@@ -98,8 +98,8 @@ async function getDetail(threadId) {
let arrStr = matcher[1];
// console.log(arrStr);
eval(`let document = { getElementById: () => {} }; var arr = ${arrStr};`);
var music = arr.music[0];
// console.log(music);
var musicArr = arr.music;
// console.log(musicArr);
} catch (e) {
console.error("解析失败", e);
return;
@@ -125,12 +125,24 @@ async function getDetail(threadId) {
};
}));
await dataManager.thread.update(threadId, {
music_title: music.title,
music_author: music.author || "",
music_url: music.url,
music_pic: music.pic || ""
});
if (musicArr.length > 1) {
console.log("典型：thread_id:", threadId);
await dataManager.thread.insertCollection(musicArr.map((music, i) => {
return {
thread_id: threadId,
music_index: i
}
}));
}
for (let i = 0; i < musicArr.length; i++) {
const music = musicArr[i];
await dataManager.thread.update(threadId, i, {
music_title: music.title,
music_author: music.author || "",
music_url: music.url,
music_pic: music.pic || ""
});
}
// console.log("done");
}
@@ -140,7 +152,7 @@ async function startFetchRealUrl() {
urlsToFetch = urlsToFetch.map(item => { return { threadId: item.thread_id, fakeUrl: item.music_url } });
for (let i = 0; i < urlsToFetch.length; i++) {
const urlToFetch = urlsToFetch[i];
console.log(`getRealUrl\t| ${i + 1}/${urlsToFetch.length} | threadId: ${urlToFetch.threadId} | ${urlToFetch.fakeUrl}`);
console.log(`getRealUrl\t| ${i + 1}/${urlsToFetch.length} | threadId: ${urlToFetch.threadId}`);
await getRealUrl(urlToFetch);
await sleepUtils.sleep(1000);
}
@@ -148,12 +160,13 @@ async function startFetchRealUrl() {
async function getRealUrl(urlToFetch) {
let { threadId, fakeUrl } = urlToFetch;
let url = "原地址已失效";
try {
let url = await requestUtils.getRedirectUrl(`https://hifini.com/${fakeUrl}`);
result = await dataManager.thread.update(threadId, { music_real_url: url });
url = await requestUtils.getRedirectUrl(`https://hifini.com/${fakeUrl}`);
} catch (e) {
console.log("重定向地址获取失败");
}
result = await dataManager.thread.update(threadId, 0, { music_real_url: url });
}
main();