1
0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

comment表添加索引

This commit is contained in:
2022-11-09 14:12:31 +08:00
parent f22a9c5228
commit f4cdef2935
3 changed files with 28 additions and 7 deletions

View File

@@ -35,7 +35,7 @@ async function fetchAll({ args = {} }) {
}
// 获取歌词详情
async function fetch({ songId, debug = false }) {
async function fetch({ songId }) {
// 首先查询有无正在爬取中的记录
var commentProgress = await dbUtils.query(`
SELECT * FROM comment_progress WHERE song_id = ? and current_status != 2 LIMIT 1
@@ -64,7 +64,7 @@ async function fetch({ songId, debug = false }) {
let isFinish = false; let pageCount = 0;
while (!isFinish) {
await global.checkIsExit();
console.log(`comment: ${songId}, page: ${++pageCount}, currentTime: ${progress.currentTime}`);
console.log(`comment: ${songId}, page: ${++pageCount}, currentTime: ${progress.currentTime} | ${new Date(progress.currentTime + 8 * 3600 * 1000).toISOString().replace("T", " ").replace("Z", "")}`);
// 是否是第一页
let isFirstPage = progress.currentStatus === 0;
@@ -75,7 +75,7 @@ async function fetch({ songId, debug = false }) {
// fs.writeFileSync(path.join(__dirname, "../../temp", `comment-${songId}-${pageCount}.json`), JSON.stringify(commentResult));
} catch (errors) {
console.error("error", errors);
await sleepUtils.sleep(3000);
await sleepUtils.sleep(10 * 1000);
continue;
}
@@ -129,7 +129,7 @@ async function fetch({ songId, debug = false }) {
progress.total = commentResult.body.total;
} else {
isFinish = true;
console.log(`comment: ${songId} done`);
console.log(`comment: ${songId} done\n`);
progress.currentStatus = 2; // 0-等待爬取/增量爬取 1-爬取中 2-完成
if (progress.maxTime == 0) { // 第一次爬取 且 没有分页的情况
progress.maxTime = comments[0] ? (comments[0].time || 0) : 0;