diff --git a/netease_music/index.js b/netease_music/index.js index 326feb3..c621ad0 100644 --- a/netease_music/index.js +++ b/netease_music/index.js @@ -1,21 +1,55 @@ const fs = require('fs'); const path = require('path'); -const dbUtils = require('../utils/dbUtils'); +// const dbUtils = require('../utils/dbUtils'); +const dbUtils = require('../utils/dbPoolUtils'); const requestUtils = require('../utils/requestUtils'); const sleepUtils = require('../utils/sleepUtils'); +dbUtils.create({ + database: "neteaseMusic", // 指定数据库 + connectionLimit: 8, // 设置数据库连接池数量 +}); + +global.dbUtils = dbUtils; const songInfoUtils = require('./src/getInfo/songInfoUtils'); const artistInfoUtils = require('./src/getInfo/artistInfoUtils'); const albumInfoUtils = require('./src/getInfo/albumInfoUtils'); +// 退出检查 +async function checkIsExit() { + if (fs.readFileSync('stop.txt') != "1") + return; + console.log(); + console.log(`收到退出指令,准备退出...`); + await sleepUtils.sleep(500); + await dbUtils.close(); + console.log(`数据库连接池已关闭`); + await sleepUtils.sleep(100); + process.exit(0); +} + +async function test() { + console.log("neteaseMusic test..."); + + let songIds = (await dbUtils.query(` + SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) + `, [])).map(song => song.song_id); + + for (let songId of songIds) { + var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`; + var json = await requestUtils.getApiResult(url); + var lyric = JSON.parse(json).lrc; + console.log(lyric); + } + // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html); +} + + async function main() { console.log("neteaseMusic Starting..."); - - // 指定数据库 - dbUtils.create("neteaseMusic"); - console.log(`${await statistics()}`); + console.log(`数据统计: ${await statistics()}`); // getMusicInfo({ songId: "1855221507" }); // getArtistInfo({ artistId: "1079074" }); @@ -28,7 +62,7 @@ async function main() { // 删除脏数据 var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []); var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []); - console.log(affectRows1.affectedRows, affectRows2.affectedRows); + console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows); await startGet(1); await sleepUtils.sleep(2000); @@ -36,7 +70,6 @@ async function main() { } async function startGet(sleepTime) { - // 从数据库中查出还缺少的歌曲,并进行爬取 console.log("start fetching songs ..."); let songIds = await dbUtils.query(` @@ -46,6 +79,7 @@ async function startGet(sleepTime) { `, []); songIds = songIds.map(item => item.song_id); for (let i = 0; i < songIds.length; i++) { + await checkIsExit(); const songId = songIds[i]; console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`); try { @@ -54,9 +88,6 @@ async function startGet(sleepTime) { console.error(err); } await sleepUtils.sleep(sleepTime); - if (fs.readFileSync('stop.txt') == "1") { - throw new Error(`Stopped`); - } } // 从数据库中查出还缺少的专辑,并进行爬取 @@ -66,6 +97,7 @@ async function startGet(sleepTime) { `, []); albumIds = albumIds.map(item => item.album_id); for (let i = 0; i < albumIds.length; i++) { + await checkIsExit(); const albumId = albumIds[i]; console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`); try { @@ -74,9 +106,6 @@ async function startGet(sleepTime) { console.error(err); } await sleepUtils.sleep(sleepTime); - if (fs.readFileSync('stop.txt') == "1") { - throw new Error(`Stopped`); - } } // 从数据库中查出还缺少的歌手,并进行爬取 @@ -86,6 +115,7 @@ async function startGet(sleepTime) { `, []); artistIds = artistIds.map(item => item.artist_id); for (let i = 0; i < artistIds.length; i++) { + await checkIsExit(); const artistId = artistIds[i]; console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`); try { @@ -94,16 +124,12 @@ async function startGet(sleepTime) { console.error(err); } await sleepUtils.sleep(sleepTime); - if (fs.readFileSync('stop.txt') == "1") { - throw new Error(`Stopped`); - } } } async function update() { console.log("neteaseMusic update ..."); - dbUtils.create("neteaseMusic"); - console.log(`${await statistics()}`); + console.log(`数据统计: ${await statistics()}`); let sleepTime = 100; @@ -114,6 +140,7 @@ async function update() { `, []); albumIds = albumIds.map(item => item.album_id); for (let i = 0; i < albumIds.length; i++) { + await checkIsExit(); const albumId = albumIds[i]; console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`); try { @@ -122,9 +149,6 @@ async function update() { console.error(err); } await sleepUtils.sleep(sleepTime); - if (fs.readFileSync('stop.txt') == "1") { - throw new Error(`Stopped`); - } } } @@ -133,28 +157,25 @@ async function statistics() { SELECT song_count, album_count, - album_v3_count, - album_v4_count, + album_v1_count, artist_count, song_album_count, song_artist_count FROM ( SELECT count(*) AS song_count FROM song ) t1, ( SELECT count(*) AS album_count FROM album ) t2, - ( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2, - ( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2, + ( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1, ( SELECT count(*) AS artist_count FROM artist ) t3, ( SELECT count(*) AS song_album_count FROM song_album_relation ) t4, ( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`; let result = await dbUtils.query(sql, []); let songCount = result[0].song_count; let albumCount = result[0].album_count; - let albumV3Count = result[0].album_v3_count; - let albumV4Count = result[0].album_v4_count; + let albumV1Count = result[0].album_v1_count; let artistCount = result[0].artist_count; let songAlbumCount = result[0].song_album_count; let songArtistCount = result[0].song_artist_count; - return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`; + return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`; } module.exports = { diff --git a/netease_music/src/getInfo/albumInfoUtils.js b/netease_music/src/getInfo/albumInfoUtils.js index 7fae769..79d0901 100644 --- a/netease_music/src/getInfo/albumInfoUtils.js +++ b/netease_music/src/getInfo/albumInfoUtils.js @@ -1,10 +1,11 @@ const fs = require('fs'); const path = require('path'); -const dbUtils = require('../../../utils/dbUtils'); const requestUtils = require('../../../utils/requestUtils'); const sleepUtils = require('../../../utils/sleepUtils'); +const dbUtils = global.dbUtils; + // 获取专辑详情 async function fetch({ albumId }) { let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]); @@ -95,7 +96,7 @@ async function fetch({ albumId }) { image: albumInfo.image, pub_date: albumInfo.pubDate, company: albumInfo.company, - version: 4 + version: 1 }); songIds.forEach(function (songId) { if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0) diff --git a/netease_music/src/getInfo/artistInfoUtils.js b/netease_music/src/getInfo/artistInfoUtils.js index 1f673e6..078fe88 100644 --- a/netease_music/src/getInfo/artistInfoUtils.js +++ b/netease_music/src/getInfo/artistInfoUtils.js @@ -1,10 +1,11 @@ const fs = require('fs'); const path = require('path'); -const dbUtils = require('../../../utils/dbUtils'); const requestUtils = require('../../../utils/requestUtils'); const sleepUtils = require('../../../utils/sleepUtils'); +const dbUtils = global.dbUtils; + // 获取音乐人详情 async function fetch({ artistId }) { let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]); diff --git a/netease_music/src/getInfo/songInfoUtils.js b/netease_music/src/getInfo/songInfoUtils.js index 8f976cf..0048647 100644 --- a/netease_music/src/getInfo/songInfoUtils.js +++ b/netease_music/src/getInfo/songInfoUtils.js @@ -1,10 +1,11 @@ const fs = require('fs'); const path = require('path'); -const dbUtils = require('../../../utils/dbUtils'); const requestUtils = require('../../../utils/requestUtils'); const sleepUtils = require('../../../utils/sleepUtils'); +const dbUtils = global.dbUtils; + // 获取音乐详情 async function fetch({ songId }) { let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]); diff --git a/netease_music/src/getInfo/userInfoUtils.js b/netease_music/src/getInfo/userInfoUtils.js index 4901126..279f449 100644 --- a/netease_music/src/getInfo/userInfoUtils.js +++ b/netease_music/src/getInfo/userInfoUtils.js @@ -1,10 +1,11 @@ const fs = require('fs'); const path = require('path'); -const dbUtils = require('../../../utils/dbUtils'); const requestUtils = require('../../../utils/requestUtils'); const sleepUtils = require('../../../utils/sleepUtils'); +const dbUtils = global.dbUtils; + // 获取用户详情 async function fetch({ userId }) { let url = `https://music.163.com/user/home?id=${userId}`; diff --git a/utils/dbPoolUtils.js b/utils/dbPoolUtils.js index 89a20fc..25330ec 100644 --- a/utils/dbPoolUtils.js +++ b/utils/dbPoolUtils.js @@ -5,9 +5,9 @@ const path = require('path'); let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8')); let pool = null; -function createPool(database) { +function create({ database, connectionLimit = 10 }) { let config = { - connectionLimit: 10, //连接数量,默认是10 + connectionLimit: connectionLimit, //连接数量,默认是10 ...globalConfig.mysql, database: database, }; @@ -21,10 +21,10 @@ function createPool(database) { // console.log('获取连接', conn.threadId); // }); - // //在连接池中建立新连接时,将触发该事件 - // pool.on('connection', function (conn) { - // console.log('建立新连接', conn.threadId); - // }); + //在连接池中建立新连接时,将触发该事件 + pool.on('connection', function (conn) { + console.log('建立新连接', conn.threadId); + }); // //等待可用连接时,将触发该事件 // pool.on('enqueue', function () { @@ -38,49 +38,45 @@ function createPool(database) { } async function query(sql, params) { - if (!pool) { - console.error('Database connection pool is not initialized yet.'); - return; - } return await new Promise(function (resolve, reject) { - //pool.query()方法可以自动的帮我们在连接池中获取可用连接 + if (!pool) { + console.error('Database connection pool is not initialized yet.'); + resolve(data); + return; + } + + // pool.query()方法可以自动的帮我们在连接池中获取可用连接 pool.query(sql, params, function (err, data) { if (err) reject(err); - // console.log(data); resolve(data); }); - // //当然我们也可以手动获取可用连接 + // // 当然我们也可以手动获取可用连接 // pool.getConnection(function (err, conn) { - // if (err) { - // throw err; - // } - // conn.query('select * from `order`', function (err, data) { - // if (err) { - // throw err; - // } - // data.forEach(function (value) { - // console.log(value.id, value.order_id, value.user_id); - // }); - - // //连接用完之后,需要释放,重新放回连接池中。 - // //注意这里并没有销毁该连接,该连接仍然可用,但需要重新获取 + // if (err) reject(err); + // conn.query(sql, function (err, data) { + // if (err) reject(err); + // resolve(data); + // // 连接用完之后,需要释放,重新放回连接池中。 + // // 注意这里并没有销毁该连接,该连接仍然可用,但需要重新获取 // conn.release(); // }); // }); }); } -function close() { - if (!pool) return; - pool.end(function (err) { - console.log('关闭连接池', err); +async function close() { + await new Promise((resolve, reject) => { + if (!pool) return; + pool.end(function (err) { + console.log('关闭连接池', err); + resolve(err); + }); }); - pool = null; } module.exports = { - createPool: createPool, + create: create, query: query, close: close, } \ No newline at end of file diff --git a/utils/dbUtils.js b/utils/dbUtils.js index 2b4dcdb..6af48f5 100644 --- a/utils/dbUtils.js +++ b/utils/dbUtils.js @@ -4,16 +4,16 @@ const fs = require('fs'); const path = require('path'); let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8')); -let database = null; +let databaseName = null; -function create(databaseName) { - database = databaseName; +function create({ database }) { + databaseName = database; } async function query(sql, params) { let config = { ...globalConfig.mysql, - database: database, + database: databaseName, }; // console.log(config); @@ -34,6 +34,8 @@ async function query(sql, params) { } module.exports = { + setConnectionLimit: () => { }, create: create, query: query, + close: () => { }, } \ No newline at end of file