update

netease_music/index.js

@@ -146,6 +146,7 @@ async function watch() {
     let statisticTimeDelta = Date.now() - watchParam.statisticTime;
 
     let statisticsString = [
+        `${new Date(Date.now() + 8 * 3600 * 1000).toISOString()}`,
         `[与上次运行统计时相比] deltaTime: ${statisticTimeDelta}ms (${(statisticTimeDelta / 1000).toFixed(2)}s)`,
         `song: ${songCount - watchParam.songCount}, album: ${albumCount - watchParam.albumCount}, artist: ${artistCount - watchParam.artistCount}, lyric: ${lyricCount - watchParam.lyricCount}, comment: ${commentCount - watchParam.commentCount}(song)/${commentTotalCount - watchParam.commentTotalCount}(comment)`,
         `[已爬取]`,
@@ -156,7 +157,7 @@ async function watch() {
         `song: ${songCount + songWaiting}, album: ${albumCount + albumWaiting}, artist: ${artistCount + artistWaiting}, lyric: ${songCount}, comment: ${songCount}`,
         `[关联关系统计]`,
         `song-album: ${songAlbumCount}, song-artist: ${songArtistCount}`,
-        `time spent: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`,
+        `sql query time: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`,
         ``
     ].join('\n');
     console.log(statisticsString);

netease_music/sql/structure.sql

@@ -53,7 +53,7 @@ CREATE TABLE `song_artist_relation` (
 
 CREATE TABLE `lyric` (
   `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
-  `version` int(10) unsigned NOT NULL COMMENT '歌词版本',
+  `version` int(10) unsigned NOT NULL COMMENT '歌词版本 -1代表没有数据',
   `lyric` text NOT NULL COMMENT '歌词',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
@@ -77,7 +77,7 @@ CREATE TABLE `comment` (
   `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
   `content` text NOT NULL COMMENT '评论内容',
   `time` varchar(50) NOT NULL DEFAULT '' COMMENT '评论时间',
-  `like_count` int(10) unsigned NOT NULL COMMENT '点赞数',
+  `like_count` int(11) NOT NULL COMMENT '点赞数',
   `comment_type` tinyint(4) unsigned NOT NULL COMMENT '评论类型 0-comments 1-hotComments 2-topComments',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',

netease_music/src/getInfo/commentUtils.js

@@ -21,7 +21,10 @@ async function fetchAll() {
 
     // 首先查询有无正在爬取中的记录
     var songIds = await dbUtils.query(`
-        SELECT song_id FROM comment_progress WHERE current_status != 2
+        -- 本机
+        SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id < 30000000 ORDER BY current_status DESC
+        -- 服务器
+        -- SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id > 30000000 ORDER BY current_status DESC
     `, []);
     songIds = songIds.map(item => item.song_id);
 
@@ -184,7 +187,7 @@ async function fetch({ songId, debug = false }) {
             current_status: progress.currentStatus,
             total: progress.total,
         }, songId]);
-        await sleepUtils.sleep(global.sleepTime);
+        // await sleepUtils.sleep(global.sleepTime);
     }
     // return commentInfo;
 }

netease_music/src/getInfo/lyricInfoUtils.js

@@ -28,7 +28,7 @@ async function fetchAll() {
 
 // 获取歌词详情
 async function fetch({ songId, debug = false }) {
-    var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
+    var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; // &kv=1&tv=-1
     try {
         // var json = fs.readFileSync(path.join(__dirname, "../../temp", `lyric-${songId}.json`), 'utf8');
         var json = await requestUtils.getApiResult(url);
@@ -45,6 +45,16 @@ async function fetch({ songId, debug = false }) {
         return;
     }
 
+    if (typeof lyric == "undefined") {
+        // 这首歌爬song的时候还在，但是现在不在了
+        dbUtils.query('INSERT IGNORE INTO lyric SET ?', {
+            song_id: songId,
+            lyric: '',
+            version: -1,
+        });
+        return;
+    }
+
     let lyricInfo = {
         songId: songId,
         lyric: lyric.lyric,

watch.js

@@ -12,7 +12,7 @@ const sleepUtils = require('./utils/sleepUtils');
 async function main() {
     do {
         await neteaseMusic.watch();
-        await sleepUtils.sleep(2000);
+        await sleepUtils.sleep(10 * 1000);
     } while (keepWatching)
 }
 main();