diff --git a/netease_music/index.js b/netease_music/index.js index 4f74c5d..81f852c 100644 --- a/netease_music/index.js +++ b/netease_music/index.js @@ -146,6 +146,7 @@ async function watch() { let statisticTimeDelta = Date.now() - watchParam.statisticTime; let statisticsString = [ + `${new Date(Date.now() + 8 * 3600 * 1000).toISOString()}`, `[与上次运行统计时相比] deltaTime: ${statisticTimeDelta}ms (${(statisticTimeDelta / 1000).toFixed(2)}s)`, `song: ${songCount - watchParam.songCount}, album: ${albumCount - watchParam.albumCount}, artist: ${artistCount - watchParam.artistCount}, lyric: ${lyricCount - watchParam.lyricCount}, comment: ${commentCount - watchParam.commentCount}(song)/${commentTotalCount - watchParam.commentTotalCount}(comment)`, `[已爬取]`, @@ -156,7 +157,7 @@ async function watch() { `song: ${songCount + songWaiting}, album: ${albumCount + albumWaiting}, artist: ${artistCount + artistWaiting}, lyric: ${songCount}, comment: ${songCount}`, `[关联关系统计]`, `song-album: ${songAlbumCount}, song-artist: ${songArtistCount}`, - `time spent: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`, + `sql query time: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`, `` ].join('\n'); console.log(statisticsString); diff --git a/netease_music/sql/structure.sql b/netease_music/sql/structure.sql index bde2c04..8f6dcaf 100644 --- a/netease_music/sql/structure.sql +++ b/netease_music/sql/structure.sql @@ -53,7 +53,7 @@ CREATE TABLE `song_artist_relation` ( CREATE TABLE `lyric` ( `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id', - `version` int(10) unsigned NOT NULL COMMENT '歌词版本', + `version` int(10) unsigned NOT NULL COMMENT '歌词版本 -1代表没有数据', `lyric` text NOT NULL COMMENT '歌词', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', @@ -77,7 +77,7 @@ CREATE TABLE `comment` ( `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id', `content` text NOT NULL COMMENT '评论内容', `time` varchar(50) NOT NULL DEFAULT '' COMMENT '评论时间', - `like_count` int(10) unsigned NOT NULL COMMENT '点赞数', + `like_count` int(11) NOT NULL COMMENT '点赞数', `comment_type` tinyint(4) unsigned NOT NULL COMMENT '评论类型 0-comments 1-hotComments 2-topComments', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间', `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间', diff --git a/netease_music/src/getInfo/commentUtils.js b/netease_music/src/getInfo/commentUtils.js index a3cfdaf..7a309ab 100644 --- a/netease_music/src/getInfo/commentUtils.js +++ b/netease_music/src/getInfo/commentUtils.js @@ -21,7 +21,10 @@ async function fetchAll() { // 首先查询有无正在爬取中的记录 var songIds = await dbUtils.query(` - SELECT song_id FROM comment_progress WHERE current_status != 2 + -- 本机 + SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id < 30000000 ORDER BY current_status DESC + -- 服务器 + -- SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id > 30000000 ORDER BY current_status DESC `, []); songIds = songIds.map(item => item.song_id); @@ -184,7 +187,7 @@ async function fetch({ songId, debug = false }) { current_status: progress.currentStatus, total: progress.total, }, songId]); - await sleepUtils.sleep(global.sleepTime); + // await sleepUtils.sleep(global.sleepTime); } // return commentInfo; } diff --git a/netease_music/src/getInfo/lyricInfoUtils.js b/netease_music/src/getInfo/lyricInfoUtils.js index 223e31c..66661f6 100644 --- a/netease_music/src/getInfo/lyricInfoUtils.js +++ b/netease_music/src/getInfo/lyricInfoUtils.js @@ -28,7 +28,7 @@ async function fetchAll() { // 获取歌词详情 async function fetch({ songId, debug = false }) { - var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; + var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; // &kv=1&tv=-1 try { // var json = fs.readFileSync(path.join(__dirname, "../../temp", `lyric-${songId}.json`), 'utf8'); var json = await requestUtils.getApiResult(url); @@ -45,6 +45,16 @@ async function fetch({ songId, debug = false }) { return; } + if (typeof lyric == "undefined") { + // 这首歌爬song的时候还在,但是现在不在了 + dbUtils.query('INSERT IGNORE INTO lyric SET ?', { + song_id: songId, + lyric: '', + version: -1, + }); + return; + } + let lyricInfo = { songId: songId, lyric: lyric.lyric, diff --git a/watch.js b/watch.js index 8bfbe78..7fe34db 100644 --- a/watch.js +++ b/watch.js @@ -12,7 +12,7 @@ const sleepUtils = require('./utils/sleepUtils'); async function main() { do { await neteaseMusic.watch(); - await sleepUtils.sleep(2000); + await sleepUtils.sleep(10 * 1000); } while (keepWatching) } main(); \ No newline at end of file