coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

comment表添加索引

Browse Source
This commit is contained in:
程序员小墨 2022-11-09 14:12:31 +08:00
parent f22a9c5228
commit f4cdef2935
3 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
netease_music/sql/neteasemusic.sql
View File

@ -1,7 +1,17 @@
/*
Source Server : MySQL 8.0
Navicat Premium Data Transfer
Source Server : localhost MySQL 8.0
Source Server Type : MySQL
Source Server Version : 80012
Source Host : localhost:3306
Source Schema : neteasemusic
Target Server Type : MySQL
Target Server Version : 80012
File Encoding : 65001
Date: 09/11/2022 13:53:17
*/
CREATE DATABASE `neteaseMusic` CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci';
@ -89,7 +99,8 @@ CREATE TABLE `comment` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`comment_id`) USING BTREE,
INDEX `song_id`(`song_id`) USING BTREE
INDEX `song_id`(`song_id`) USING BTREE,
INDEX `user_id`(`user_id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci ROW_FORMAT = Dynamic;
-- ----------------------------
@ -289,6 +300,15 @@ CREATE TABLE `wait_check_artist` (
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for wait_check_comment
-- ----------------------------
DROP TABLE IF EXISTS `wait_check_comment`;
CREATE TABLE `wait_check_comment` (
`id` bigint(20) UNSIGNED NOT NULL COMMENT 'id',
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for wait_check_lyric
-- ----------------------------

3
netease_music/sql/structure.sql
View File

@ -166,7 +166,8 @@ CREATE TABLE `comment` (
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`comment_id`),
KEY `song_id` (`song_id`)
KEY `song_id` (`song_id`),
KEY `user_id`(`user_id`)
);
CREATE TABLE `comment_progress` (

8
netease_music/src/getInfo/commentUtils.js
View File

@ -35,7 +35,7 @@ async function fetchAll({ args = {} }) {
}
// 获取歌词详情
async function fetch({ songId, debug = false }) {
async function fetch({ songId }) {
// 首先查询有无正在爬取中的记录
var commentProgress = await dbUtils.query(`
SELECT * FROM comment_progress WHERE song_id = ? and current_status != 2 LIMIT 1
@ -64,7 +64,7 @@ async function fetch({ songId, debug = false }) {
let isFinish = false; let pageCount = 0;
while (!isFinish) {
await global.checkIsExit();
console.log(`comment: ${songId}, page: ${++pageCount}, currentTime: ${progress.currentTime}`);
console.log(`comment: ${songId}, page: ${++pageCount}, currentTime: ${progress.currentTime} | ${new Date(progress.currentTime + 8 * 3600 * 1000).toISOString().replace("T", " ").replace("Z", "")}`);
// 是否是第一页
let isFirstPage = progress.currentStatus === 0;
@ -75,7 +75,7 @@ async function fetch({ songId, debug = false }) {
// fs.writeFileSync(path.join(__dirname, "../../temp", `comment-${songId}-${pageCount}.json`), JSON.stringify(commentResult));
} catch (errors) {
console.error("error", errors);
await sleepUtils.sleep(3000);
await sleepUtils.sleep(10 * 1000);
continue;
}
@ -129,7 +129,7 @@ async function fetch({ songId, debug = false }) {
progress.total = commentResult.body.total;
} else {
isFinish = true;
console.log(`comment: ${songId} done`);
console.log(`comment: ${songId} done\n`);
progress.currentStatus = 2; // 0-等待爬取/增量爬取 1-爬取中 2-完成
if (progress.maxTime == 0) { // 第一次爬取 且 没有分页的情况
progress.maxTime = comments[0] ? (comments[0].time || 0) : 0;