const fs = require('fs'); const path = require('path'); const dbUtils = require(global.useMysqlPool ? '../utils/dbPoolUtils' : '../utils/dbUtils'); const requestUtils = require('../utils/requestUtils'); const sleepUtils = require('../utils/sleepUtils'); dbUtils.create({ database: "neteaseMusic", // 指定数据库 connectionLimit: 8, // 设置数据库连接池数量 }); global.dbUtils = dbUtils; const songInfoUtils = require('./src/getInfo/songInfoUtils'); const artistInfoUtils = require('./src/getInfo/artistInfoUtils'); const albumInfoUtils = require('./src/getInfo/albumInfoUtils'); console.log("global.useMysqlPool:", !!global.useMysqlPool); // 退出检查 async function checkIsExit() { if (fs.readFileSync('stop.txt') != "1") return; console.log(); console.log(`收到退出指令,准备退出...`); await sleepUtils.sleep(500); await dbUtils.close(); console.log(`数据库连接池已关闭`); await sleepUtils.sleep(100); process.exit(0); } async function test() { console.log("neteaseMusic test..."); let songIds = (await dbUtils.query(` SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) `, [])).map(song => song.song_id); for (let songId of songIds) { var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; var json = await requestUtils.getApiResult(url); var lyric = JSON.parse(json).lrc; console.log(lyric); } // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html); } async function main() { console.log("neteaseMusic Starting..."); console.log(`数据统计: ${await statistics()}`); // getMusicInfo({ songId: "1855221507" }); // getArtistInfo({ artistId: "1079074" }); // getAlbumInfo({ albumId: "74268047" }); // 不是所有歌手都有个人主页 例如 https://music.163.com/#/artist?id=1079075 // getUserInfo({ userId: "37365202" }); while (true) { // 删除脏数据 var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []); var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []); console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows); await startGet(1); await sleepUtils.sleep(2000); } } async function startGet(sleepTime) { // 从数据库中查出还缺少的歌曲,并进行爬取 console.log("start fetching songs ..."); let songIds = await dbUtils.query(` SELECT DISTINCT song_id FROM song_artist_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) UNION SELECT DISTINCT song_id FROM song_album_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) `, []); songIds = songIds.map(item => item.song_id); for (let i = 0; i < songIds.length; i++) { await checkIsExit(); const songId = songIds[i]; console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`); try { await songInfoUtils.fetch({ songId: songId }); } catch (err) { console.error(err); } await sleepUtils.sleep(sleepTime); } // 从数据库中查出还缺少的专辑,并进行爬取 console.log("start fetching albums ...") let albumIds = await dbUtils.query(` SELECT DISTINCT album_id FROM song_album_relation WHERE album_id NOT IN ( SELECT DISTINCT album_id FROM album ) `, []); albumIds = albumIds.map(item => item.album_id); for (let i = 0; i < albumIds.length; i++) { await checkIsExit(); const albumId = albumIds[i]; console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`); try { await albumInfoUtils.fetch({ albumId: albumId }); } catch (err) { console.error(err); } await sleepUtils.sleep(sleepTime); } // 从数据库中查出还缺少的歌手,并进行爬取 console.log("start fetching artists ...") let artistIds = await dbUtils.query(` SELECT DISTINCT artist_id FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist ) `, []); artistIds = artistIds.map(item => item.artist_id); for (let i = 0; i < artistIds.length; i++) { await checkIsExit(); const artistId = artistIds[i]; console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`); try { await artistInfoUtils.fetch({ artistId: artistId }); } catch (err) { console.error(err); } await sleepUtils.sleep(sleepTime); } } async function update() { console.log("neteaseMusic update ..."); console.log(`数据统计: ${await statistics()}`); let sleepTime = 100; // 从数据库中查出现有专辑,并进行更新 console.log("start fetching albums ...") let albumIds = await dbUtils.query(` SELECT DISTINCT album_id FROM album WHERE version = 1 -- and description like '%专辑《%》,简介:%' `, []); albumIds = albumIds.map(item => item.album_id); for (let i = 0; i < albumIds.length; i++) { await checkIsExit(); const albumId = albumIds[i]; console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`); try { await albumInfoUtils.update({ albumId: albumId }); } catch (err) { console.error(err); } await sleepUtils.sleep(sleepTime); } } async function statistics() { let sql = ` SELECT song_count, album_count, album_v1_count, artist_count, song_album_count, song_artist_count FROM ( SELECT count(*) AS song_count FROM song ) t1, ( SELECT count(*) AS album_count FROM album ) t2, ( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1, ( SELECT count(*) AS artist_count FROM artist ) t3, ( SELECT count(*) AS song_album_count FROM song_album_relation ) t4, ( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`; let result = await dbUtils.query(sql, []); let songCount = result[0].song_count; let albumCount = result[0].album_count; let albumV1Count = result[0].album_v1_count; let artistCount = result[0].artist_count; let songAlbumCount = result[0].song_album_count; let songArtistCount = result[0].song_artist_count; return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`; } module.exports = { main: main, update: update, test: test, }