// 引入modules const fs = require('fs'); const path = require('path'); const dbUtils = require(global.useMysqlPool ? '../utils/dbPoolUtils' : '../utils/dbUtils'); const sleepUtils = require('../utils/sleepUtils'); // 数据库连接池 dbUtils.create({ database: "neteaseMusic", // 指定数据库 connectionLimit: global.connectionLimit || 10, // 设置数据库连接池数量 }); global.dbUtils = dbUtils; console.log("global.useMysqlPool:", !!global.useMysqlPool); // 两次请求之间停顿时间 global.sleepTime = 300; // 引入utils const songInfoUtils = require('./src/getInfo/songInfoUtils'); const artistInfoUtils = require('./src/getInfo/artistInfoUtils'); const albumInfoUtils = require('./src/getInfo/albumInfoUtils'); const lyricInfoUtils = require('./src/getInfo/lyricInfoUtils'); const commentUtils = require('./src/getInfo/commentUtils'); /** * 测试 */ async function test() { console.log("neteaseMusic test..."); // 不是所有歌手都有个人主页 例如 https://music.163.com/#/artist?id=1079075 // let res = await albumInfoUtils.fetch({ albumId: "9156", debug: true }); // let res = await artistInfoUtils.fetch({ artistId: "12023508" }); // let res = await songInfoUtils.fetch({ songId: "437608327" }); // let res = await albumInfoUtils.getFromDatabase({ albumId: "9156" }); // let res = await artistInfoUtils.getFromDatabase({ artistId: "12023508" }); // let res = await songInfoUtils.getFromDatabase({ songId: "437608327" }); console.log(res); } /** * 主函数 */ async function main() { console.log("neteaseMusic Start fetch ..."); while (true) { // // 删除脏数据 // var affectedRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []); // var affectedRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []); // console.log(`删除脏数据 affectedRows:`, affectedRows1.affectedRows, affectedRows2.affectedRows); await songInfoUtils.fetchAll(); await albumInfoUtils.fetchAll({}); await artistInfoUtils.fetchAll(); await lyricInfoUtils.fetchAll(); await commentUtils.fetchAll(); await sleepUtils.sleep(2000); } } /** * 数据更新 (重新爬取) */ async function update() { console.log("neteaseMusic Start update ..."); while (true) { await albumInfoUtils.fetchAll({ isUpdate: true }); await sleepUtils.sleep(2000); } } /** * 统计数据库中数据 */ let watchParam = { songCount: 0, albumCount: 0, artistCount: 0, lyricCount: 0, commentCount: 0, commentTotalCount: 0, }; async function watch() { let sql = ` SELECT song_count, song_waiting_1 + song_waiting_2 as song_waiting, album_count, album_waiting, artist_count, artist_waiting, lyric_count, comment_count, comment_total_count, song_album_count, song_artist_count FROM ( SELECT count(*) AS song_count FROM song ) t_song, ( SELECT count( DISTINCT song_id ) as song_waiting_1 FROM song_artist_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) ) t_song_waiting_song_artist, ( SELECT count( DISTINCT song_id ) as song_waiting_2 FROM song_album_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) ) t_song_waiting_song_album, ( SELECT count(*) AS album_count FROM album ) t_album, ( SELECT count( DISTINCT album_id ) as album_waiting FROM song_album_relation WHERE album_id NOT IN ( SELECT DISTINCT album_id FROM album ) ) as t_album_waiting_song_album, ( SELECT count(*) AS artist_count FROM artist ) t_artist, ( SELECT count( DISTINCT artist_id ) as artist_waiting FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist ) ) as t_album_waiting_song_artist, ( SELECT count(*) AS lyric_count FROM lyric ) t_lyric, ( SELECT count( DISTINCT song_id ) AS comment_count, count( comment_id ) AS comment_total_count FROM comment ) t_comment, ( SELECT count(*) AS song_album_count FROM song_album_relation ) t_song_album, ( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t_song_artist `; console.log("开始统计 ..."); let startTime = Date.now(); let result = await dbUtils.query(sql, []); let timeSpent = Date.now() - startTime; let songCount = result[0].song_count; let songWaiting = result[0].song_waiting; let albumCount = result[0].album_count; let albumWaiting = result[0].album_waiting; let artistCount = result[0].artist_count; let artistWaiting = result[0].artist_waiting; let lyricCount = result[0].lyric_count; let commentCount = result[0].comment_count; let commentTotalCount = result[0].comment_total_count; let songAlbumCount = result[0].song_album_count; let songArtistCount = result[0].song_artist_count; let statisticsString = [ `[与上次运行统计时相比]`, `song: ${songCount - watchParam.songCount}, album: ${albumCount - watchParam.albumCount}, artist: ${artistCount - watchParam.artistCount}, lyric: ${lyricCount - watchParam.lyricCount}, comment: ${commentCount - watchParam.commentCount}(song)/${commentTotalCount - watchParam.commentTotalCount}(comment)`, `[已爬取]`, `song: ${songCount}, album: ${albumCount}, artist: ${artistCount}, lyric: ${lyricCount}, comment: ${commentCount}(song)/${commentTotalCount}(comment)`, `[待爬取]`, `song: ${songWaiting}, album: ${albumWaiting}, artist: ${artistWaiting}, lyric: ${songCount - lyricCount}, comment: ${songCount - commentCount}`, `[总计] (已爬取 + 待爬取)`, `song: ${songCount + songWaiting}, album: ${albumCount + albumWaiting}, artist: ${artistCount + artistWaiting}, lyric: ${songCount}, comment: ${songCount}`, `[关联关系统计]`, `song-album: ${songAlbumCount}, song-artist: ${songArtistCount}`, `time spent: ${timeSpent}ms(${(timeSpent / 1000).toFixed(2)}s)`, `` ].join('\n'); console.log(statisticsString); watchParam = { songCount: songCount, albumCount: albumCount, artistCount: artistCount, lyricCount: lyricCount, commentCount: commentCount, commentTotalCount: commentTotalCount, } } /** * 退出程序 */ global.checkIsExit = async function () { if (fs.readFileSync('stop.txt') != "1") return; console.log(); console.log(`收到退出指令,准备退出...`); await sleepUtils.sleep(500); await dbUtils.close(); console.log(`数据库连接池已关闭`); await sleepUtils.sleep(100); process.exit(0); } module.exports = { main: main, update: update, watch: watch, test: test, }