195 lines
7.4 KiB
JavaScript
195 lines
7.4 KiB
JavaScript
// 引入modules
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const dbUtils = require(global.useMysqlPool ? '../utils/dbPoolUtils' : '../utils/dbUtils');
|
|
const sleepUtils = require('../utils/sleepUtils');
|
|
|
|
// 数据库连接池
|
|
dbUtils.create({
|
|
database: "neteaseMusic", // 指定数据库
|
|
connectionLimit: global.connectionLimit || 10, // 设置数据库连接池数量
|
|
});
|
|
global.dbUtils = dbUtils;
|
|
console.log("global.useMysqlPool:", !!global.useMysqlPool);
|
|
|
|
// 两次请求之间停顿时间
|
|
global.sleepTime = 300;
|
|
|
|
// 引入utils
|
|
const songInfoUtils = require('./src/getInfo/songInfoUtils');
|
|
const artistInfoUtils = require('./src/getInfo/artistInfoUtils');
|
|
const albumInfoUtils = require('./src/getInfo/albumInfoUtils');
|
|
const lyricInfoUtils = require('./src/getInfo/lyricInfoUtils');
|
|
const commentUtils = require('./src/getInfo/commentUtils');
|
|
|
|
/**
|
|
* 测试
|
|
*/
|
|
async function test() {
|
|
console.log("neteaseMusic test...");
|
|
|
|
// 不是所有歌手都有个人主页 例如 https://music.163.com/#/artist?id=1079075
|
|
|
|
// let res = await albumInfoUtils.fetch({ albumId: "9156", debug: true });
|
|
// let res = await artistInfoUtils.fetch({ artistId: "12023508" });
|
|
// let res = await songInfoUtils.fetch({ songId: "437608327" });
|
|
|
|
// let res = await albumInfoUtils.getFromDatabase({ albumId: "9156" });
|
|
// let res = await artistInfoUtils.getFromDatabase({ artistId: "12023508" });
|
|
// let res = await songInfoUtils.getFromDatabase({ songId: "437608327" });
|
|
|
|
console.log(res);
|
|
}
|
|
|
|
/**
|
|
* 主函数
|
|
*/
|
|
async function main() {
|
|
console.log("neteaseMusic Start fetch ...");
|
|
while (true) {
|
|
// // 删除脏数据
|
|
// var affectedRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []);
|
|
// var affectedRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
|
|
// console.log(`删除脏数据 affectedRows:`, affectedRows1.affectedRows, affectedRows2.affectedRows);
|
|
|
|
await songInfoUtils.fetchAll();
|
|
await albumInfoUtils.fetchAll({});
|
|
await artistInfoUtils.fetchAll();
|
|
await lyricInfoUtils.fetchAll();
|
|
await commentUtils.fetchAll();
|
|
await sleepUtils.sleep(2000);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 数据更新 (重新爬取)
|
|
*/
|
|
async function update() {
|
|
console.log("neteaseMusic Start update ...");
|
|
while (true) {
|
|
await albumInfoUtils.fetchAll({ isUpdate: true });
|
|
await sleepUtils.sleep(2000);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 统计数据库中数据
|
|
*/
|
|
let watchParam = {
|
|
statisticTime: Date.now(),
|
|
songCount: 0,
|
|
albumCount: 0,
|
|
artistCount: 0,
|
|
lyricCount: 0,
|
|
commentCount: 0,
|
|
commentTotalCount: 0,
|
|
};
|
|
async function watch() {
|
|
let sql = `
|
|
SELECT
|
|
song_count,
|
|
song_waiting_1 + song_waiting_2 as song_waiting,
|
|
|
|
album_count,
|
|
album_waiting,
|
|
|
|
artist_count,
|
|
artist_waiting,
|
|
|
|
lyric_count,
|
|
|
|
comment_count,
|
|
comment_total_count,
|
|
|
|
song_album_count,
|
|
song_artist_count
|
|
FROM
|
|
( SELECT count(*) AS song_count FROM song ) t_song,
|
|
( SELECT count( DISTINCT song_id ) as song_waiting_1 FROM song_artist_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) ) t_song_waiting_song_artist,
|
|
( SELECT count( DISTINCT song_id ) as song_waiting_2 FROM song_album_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) ) t_song_waiting_song_album,
|
|
|
|
( SELECT count(*) AS album_count FROM album ) t_album,
|
|
( SELECT count( DISTINCT album_id ) as album_waiting FROM song_album_relation WHERE album_id NOT IN ( SELECT DISTINCT album_id FROM album ) ) as t_album_waiting_song_album,
|
|
|
|
( SELECT count(*) AS artist_count FROM artist ) t_artist,
|
|
( SELECT count( DISTINCT artist_id ) as artist_waiting FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist ) ) as t_album_waiting_song_artist,
|
|
|
|
( SELECT count(*) AS lyric_count FROM lyric ) t_lyric,
|
|
|
|
( SELECT count( DISTINCT song_id ) AS comment_count, count( comment_id ) AS comment_total_count FROM comment ) t_comment,
|
|
|
|
( SELECT count(*) AS song_album_count FROM song_album_relation ) t_song_album,
|
|
( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t_song_artist
|
|
`;
|
|
console.log("开始统计 ...");
|
|
let startTime = Date.now();
|
|
let result = await dbUtils.query(sql, []);
|
|
let timeSpent = Date.now() - startTime;
|
|
|
|
let songCount = result[0].song_count;
|
|
let songWaiting = result[0].song_waiting;
|
|
|
|
let albumCount = result[0].album_count;
|
|
let albumWaiting = result[0].album_waiting;
|
|
|
|
let artistCount = result[0].artist_count;
|
|
let artistWaiting = result[0].artist_waiting;
|
|
|
|
let lyricCount = result[0].lyric_count;
|
|
|
|
let commentCount = result[0].comment_count;
|
|
let commentTotalCount = result[0].comment_total_count;
|
|
|
|
let songAlbumCount = result[0].song_album_count;
|
|
let songArtistCount = result[0].song_artist_count;
|
|
|
|
let statisticTimeDelta = Date.now() - watchParam.statisticTime;
|
|
|
|
let statisticsString = [
|
|
`${new Date(Date.now() + 8 * 3600 * 1000).toISOString()}`,
|
|
`[与上次运行统计时相比] deltaTime: ${statisticTimeDelta}ms (${(statisticTimeDelta / 1000).toFixed(2)}s)`,
|
|
`song: ${songCount - watchParam.songCount}, album: ${albumCount - watchParam.albumCount}, artist: ${artistCount - watchParam.artistCount}, lyric: ${lyricCount - watchParam.lyricCount}, comment: ${commentCount - watchParam.commentCount}(song)/${commentTotalCount - watchParam.commentTotalCount}(comment)`,
|
|
`[已爬取]`,
|
|
`song: ${songCount}, album: ${albumCount}, artist: ${artistCount}, lyric: ${lyricCount}, comment: ${commentCount}(song)/${commentTotalCount}(comment)`,
|
|
`[待爬取]`,
|
|
`song: ${songWaiting}, album: ${albumWaiting}, artist: ${artistWaiting}, lyric: ${songCount - lyricCount}, comment: ${songCount - commentCount}`,
|
|
`[总计] (已爬取 + 待爬取)`,
|
|
`song: ${songCount + songWaiting}, album: ${albumCount + albumWaiting}, artist: ${artistCount + artistWaiting}, lyric: ${songCount}, comment: ${songCount}`,
|
|
`[关联关系统计]`,
|
|
`song-album: ${songAlbumCount}, song-artist: ${songArtistCount}`,
|
|
`sql query time: ${timeSpent}ms (${(timeSpent / 1000).toFixed(2)}s)`,
|
|
``
|
|
].join('\n');
|
|
console.log(statisticsString);
|
|
watchParam = {
|
|
statisticTime: Date.now(),
|
|
songCount: songCount,
|
|
albumCount: albumCount,
|
|
artistCount: artistCount,
|
|
lyricCount: lyricCount,
|
|
commentCount: commentCount,
|
|
commentTotalCount: commentTotalCount,
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 退出程序
|
|
*/
|
|
global.checkIsExit = async function () {
|
|
if (fs.readFileSync('stop.txt') != "1")
|
|
return;
|
|
console.log();
|
|
console.log(`收到退出指令，准备退出...`);
|
|
await sleepUtils.sleep(500);
|
|
await dbUtils.close();
|
|
console.log(`数据库连接池已关闭`);
|
|
await sleepUtils.sleep(100);
|
|
process.exit(0);
|
|
}
|
|
|
|
module.exports = {
|
|
main: main,
|
|
update: update,
|
|
watch: watch,
|
|
test: test,
|
|
} |