coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

update

Browse Source
This commit is contained in:
程序员小墨 2022-10-03 14:32:04 +08:00
parent e778b455ed
commit b4d63489c3
5 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
netease_music/index.js
View File

@ -146,6 +146,7 @@ async function watch() {
let statisticTimeDelta = Date.now() - watchParam.statisticTime;
let statisticsString = [
`${new Date(Date.now() + 8 * 3600 * 1000).toISOString()}`,
`[与上次运行统计时相比] deltaTime: ${statisticTimeDelta}ms (${(statisticTimeDelta / 1000).toFixed(2)}s)`,
`song: ${songCount - watchParam.songCount}, album: ${albumCount - watchParam.albumCount}, artist: ${artistCount - watchParam.artistCount}, lyric: ${lyricCount - watchParam.lyricCount}, comment: ${commentCount - watchParam.commentCount}(song)/${commentTotalCount - watchParam.commentTotalCount}(comment)`,
`[已爬取]`,
@ -156,7 +157,7 @@ async function watch() {
`song: ${songCount + songWaiting}, album: ${albumCount + albumWaiting}, artist: ${artistCount + artistWaiting}, lyric: ${songCount}, comment: ${songCount}`,
`[关联关系统计]`,
`song-album: ${songAlbumCount}, song-artist: ${songArtistCount}`,
`time spent: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`,
`sql query time: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`,
``
].join('\n');
console.log(statisticsString);

4
netease_music/sql/structure.sql
View File

@ -53,7 +53,7 @@ CREATE TABLE `song_artist_relation` (
CREATE TABLE `lyric` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
`version` int(10) unsigned NOT NULL COMMENT '歌词版本',
`version` int(10) unsigned NOT NULL COMMENT '歌词版本 -1代表没有数据',
`lyric` text NOT NULL COMMENT '歌词',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
@ -77,7 +77,7 @@ CREATE TABLE `comment` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
`content` text NOT NULL COMMENT '评论内容',
`time` varchar(50) NOT NULL DEFAULT '' COMMENT '评论时间',
`like_count` int(10) unsigned NOT NULL COMMENT '点赞数',
`like_count` int(11) NOT NULL COMMENT '点赞数',
`comment_type` tinyint(4) unsigned NOT NULL COMMENT '评论类型 0-comments 1-hotComments 2-topComments',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',

7
netease_music/src/getInfo/commentUtils.js
View File

@ -21,7 +21,10 @@ async function fetchAll() {
// 首先查询有无正在爬取中的记录
var songIds = await dbUtils.query(`
SELECT song_id FROM comment_progress WHERE current_status != 2
-- 本机
SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id < 30000000 ORDER BY current_status DESC
-- 服务器
-- SELECT song_id FROM comment_progress WHERE current_status != 2 AND song_id > 30000000 ORDER BY current_status DESC
`, []);
songIds = songIds.map(item => item.song_id);
@ -184,7 +187,7 @@ async function fetch({ songId, debug = false }) {
current_status: progress.currentStatus,
total: progress.total,
}, songId]);
await sleepUtils.sleep(global.sleepTime);
// await sleepUtils.sleep(global.sleepTime);
}
// return commentInfo;
}

12
netease_music/src/getInfo/lyricInfoUtils.js
View File

@ -28,7 +28,7 @@ async function fetchAll() {
// 获取歌词详情
async function fetch({ songId, debug = false }) {
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; // &kv=1&tv=-1
try {
// var json = fs.readFileSync(path.join(__dirname, "../../temp", `lyric-${songId}.json`), 'utf8');
var json = await requestUtils.getApiResult(url);
@ -45,6 +45,16 @@ async function fetch({ songId, debug = false }) {
return;
}
if (typeof lyric == "undefined") {
// 这首歌爬song的时候还在，但是现在不在了
dbUtils.query('INSERT IGNORE INTO lyric SET ?', {
song_id: songId,
lyric: '',
version: -1,
});
return;
}
let lyricInfo = {
songId: songId,
lyric: lyric.lyric,

2
watch.js
View File

@ -12,7 +12,7 @@ const sleepUtils = require('./utils/sleepUtils');
async function main() {
do {
await neteaseMusic.watch();
await sleepUtils.sleep(2000);
await sleepUtils.sleep(10 * 1000);
} while (keepWatching)
}
main();