使用连接池;退出检查优化

netease_music/index.js

@@ -1,21 +1,55 @@
 const fs = require('fs');
 const path = require('path');
-const dbUtils = require('../utils/dbUtils');
+// const dbUtils = require('../utils/dbUtils');
+const dbUtils = require('../utils/dbPoolUtils');
 
 const requestUtils = require('../utils/requestUtils');
 const sleepUtils = require('../utils/sleepUtils');
 
+dbUtils.create({
+    database: "neteaseMusic",   // 指定数据库
+    connectionLimit: 8,         // 设置数据库连接池数量
+});
+
+global.dbUtils = dbUtils;
 const songInfoUtils = require('./src/getInfo/songInfoUtils');
 const artistInfoUtils = require('./src/getInfo/artistInfoUtils');
 const albumInfoUtils = require('./src/getInfo/albumInfoUtils');
 
 
+// 退出检查
+async function checkIsExit() {
+    if (fs.readFileSync('stop.txt') != "1")
+        return;
+    console.log();
+    console.log(`收到退出指令，准备退出...`);
+    await sleepUtils.sleep(500);
+    await dbUtils.close();
+    console.log(`数据库连接池已关闭`);
+    await sleepUtils.sleep(100);
+    process.exit(0);
+}
+
+async function test() {
+    console.log("neteaseMusic test...");
+
+    let songIds = (await dbUtils.query(`
+        SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
+    `, [])).map(song => song.song_id);
+
+    for (let songId of songIds) {
+        var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
+        var json = await requestUtils.getApiResult(url);
+        var lyric = JSON.parse(json).lrc;
+        console.log(lyric);
+    }
+    // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
+}
+
+
 async function main() {
     console.log("neteaseMusic Starting...");
-
+    console.log(`数据统计: ${await statistics()}`);
-    // 指定数据库
-    dbUtils.create("neteaseMusic");
-    console.log(`${await statistics()}`);
 
     // getMusicInfo({ songId: "1855221507" });
     // getArtistInfo({ artistId: "1079074" });
@@ -28,7 +62,7 @@ async function main() {
         // 删除脏数据
         var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []);
         var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
-        console.log(affectRows1.affectedRows, affectRows2.affectedRows);
+        console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows);
 
         await startGet(1);
         await sleepUtils.sleep(2000);
@@ -36,7 +70,6 @@ async function main() {
 }
 
 async function startGet(sleepTime) {
-
     // 从数据库中查出还缺少的歌曲，并进行爬取
     console.log("start fetching songs ...");
     let songIds = await dbUtils.query(`
@@ -46,6 +79,7 @@ async function startGet(sleepTime) {
     `, []);
     songIds = songIds.map(item => item.song_id);
     for (let i = 0; i < songIds.length; i++) {
+        await checkIsExit();
         const songId = songIds[i];
         console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`);
         try {
@@ -54,9 +88,6 @@ async function startGet(sleepTime) {
             console.error(err);
         }
         await sleepUtils.sleep(sleepTime);
-        if (fs.readFileSync('stop.txt') == "1") {
-            throw new Error(`Stopped`);
-        }
     }
 
     // 从数据库中查出还缺少的专辑，并进行爬取
@@ -66,6 +97,7 @@ async function startGet(sleepTime) {
     `, []);
     albumIds = albumIds.map(item => item.album_id);
     for (let i = 0; i < albumIds.length; i++) {
+        await checkIsExit();
         const albumId = albumIds[i];
         console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
         try {
@@ -74,9 +106,6 @@ async function startGet(sleepTime) {
             console.error(err);
         }
         await sleepUtils.sleep(sleepTime);
-        if (fs.readFileSync('stop.txt') == "1") {
-            throw new Error(`Stopped`);
-        }
     }
 
     // 从数据库中查出还缺少的歌手，并进行爬取
@@ -86,6 +115,7 @@ async function startGet(sleepTime) {
     `, []);
     artistIds = artistIds.map(item => item.artist_id);
     for (let i = 0; i < artistIds.length; i++) {
+        await checkIsExit();
         const artistId = artistIds[i];
         console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`);
         try {
@@ -94,16 +124,12 @@ async function startGet(sleepTime) {
             console.error(err);
         }
         await sleepUtils.sleep(sleepTime);
-        if (fs.readFileSync('stop.txt') == "1") {
-            throw new Error(`Stopped`);
-        }
     }
 }
 
 async function update() {
     console.log("neteaseMusic update ...");
-    dbUtils.create("neteaseMusic");
+    console.log(`数据统计: ${await statistics()}`);
-    console.log(`${await statistics()}`);
 
     let sleepTime = 100;
 
@@ -114,6 +140,7 @@ async function update() {
     `, []);
     albumIds = albumIds.map(item => item.album_id);
     for (let i = 0; i < albumIds.length; i++) {
+        await checkIsExit();
         const albumId = albumIds[i];
         console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
         try {
@@ -122,9 +149,6 @@ async function update() {
             console.error(err);
         }
         await sleepUtils.sleep(sleepTime);
-        if (fs.readFileSync('stop.txt') == "1") {
-            throw new Error(`Stopped`);
-        }
     }
 }
 
@@ -133,28 +157,25 @@ async function statistics() {
         SELECT
             song_count,
             album_count,
-            album_v3_count,
+            album_v1_count,
-            album_v4_count,
             artist_count,
             song_album_count,
             song_artist_count
         FROM
             ( SELECT count(*) AS song_count FROM song ) t1,
             ( SELECT count(*) AS album_count FROM album ) t2,
-            ( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
+            ( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1,
-            ( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
             ( SELECT count(*) AS artist_count FROM artist ) t3,
             ( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
             ( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
     let result = await dbUtils.query(sql, []);
     let songCount = result[0].song_count;
     let albumCount = result[0].album_count;
-    let albumV3Count = result[0].album_v3_count;
+    let albumV1Count = result[0].album_v1_count;
-    let albumV4Count = result[0].album_v4_count;
     let artistCount = result[0].artist_count;
     let songAlbumCount = result[0].song_album_count;
     let songArtistCount = result[0].song_artist_count;
-    return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
+    return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
 }
 
 module.exports = {

netease_music/src/getInfo/albumInfoUtils.js

@@ -1,10 +1,11 @@
 const fs = require('fs');
 const path = require('path');
-const dbUtils = require('../../../utils/dbUtils');
 
 const requestUtils = require('../../../utils/requestUtils');
 const sleepUtils = require('../../../utils/sleepUtils');
 
+const dbUtils = global.dbUtils;
+
 // 获取专辑详情
 async function fetch({ albumId }) {
     let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
@@ -95,7 +96,7 @@ async function fetch({ albumId }) {
         image: albumInfo.image,
         pub_date: albumInfo.pubDate,
         company: albumInfo.company,
-        version: 4
+        version: 1
     });
     songIds.forEach(function (songId) {
         if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0)

netease_music/src/getInfo/artistInfoUtils.js

@@ -1,10 +1,11 @@
 const fs = require('fs');
 const path = require('path');
-const dbUtils = require('../../../utils/dbUtils');
 
 const requestUtils = require('../../../utils/requestUtils');
 const sleepUtils = require('../../../utils/sleepUtils');
 
+const dbUtils = global.dbUtils;
+
 // 获取音乐人详情
 async function fetch({ artistId }) {
     let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);

netease_music/src/getInfo/songInfoUtils.js

@@ -1,10 +1,11 @@
 const fs = require('fs');
 const path = require('path');
-const dbUtils = require('../../../utils/dbUtils');
 
 const requestUtils = require('../../../utils/requestUtils');
 const sleepUtils = require('../../../utils/sleepUtils');
 
+const dbUtils = global.dbUtils;
+
 // 获取音乐详情
 async function fetch({ songId }) {
     let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]);

netease_music/src/getInfo/userInfoUtils.js

@@ -1,10 +1,11 @@
 const fs = require('fs');
 const path = require('path');
-const dbUtils = require('../../../utils/dbUtils');
 
 const requestUtils = require('../../../utils/requestUtils');
 const sleepUtils = require('../../../utils/sleepUtils');
 
+const dbUtils = global.dbUtils;
+
 // 获取用户详情
 async function fetch({ userId }) {
     let url = `https://music.163.com/user/home?id=${userId}`;

utils/dbPoolUtils.js

@@ -5,9 +5,9 @@ const path = require('path');
 let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8'));
 let pool = null;
 
-function createPool(database) {
+function create({ database, connectionLimit = 10 }) {
     let config = {
-        connectionLimit: 10, //连接数量，默认是10
+        connectionLimit: connectionLimit, //连接数量，默认是10
         ...globalConfig.mysql,
         database: database,
     };
@@ -21,10 +21,10 @@ function createPool(database) {
     //     console.log('获取连接', conn.threadId);
     // });
 
-    // //在连接池中建立新连接时，将触发该事件
+    //在连接池中建立新连接时，将触发该事件
-    // pool.on('connection', function (conn) {
+    pool.on('connection', function (conn) {
-    //     console.log('建立新连接', conn.threadId);
+        console.log('建立新连接', conn.threadId);
-    // });
+    });
 
     // //等待可用连接时，将触发该事件
     // pool.on('enqueue', function () {
@@ -38,31 +38,25 @@ function createPool(database) {
 }
 
 async function query(sql, params) {
+    return await new Promise(function (resolve, reject) {
         if (!pool) {
             console.error('Database connection pool is not initialized yet.');
+            resolve(data);
             return;
         }
-    return await new Promise(function (resolve, reject) {
+
         // pool.query()方法可以自动的帮我们在连接池中获取可用连接
         pool.query(sql, params, function (err, data) {
             if (err) reject(err);
-            // console.log(data);
             resolve(data);
         });
 
         // // 当然我们也可以手动获取可用连接
         // pool.getConnection(function (err, conn) {
-        //     if (err) {
+        //     if (err) reject(err);
-        //         throw err;
+        //     conn.query(sql, function (err, data) {
-        //     }
+        //         if (err) reject(err);
-        //     conn.query('select * from `order`', function (err, data) {
+        //         resolve(data);
-        //         if (err) {
-        //             throw err;
-        //         }
-        //         data.forEach(function (value) {
-        //             console.log(value.id, value.order_id, value.user_id);
-        //         });
-
         //         // 连接用完之后，需要释放，重新放回连接池中。
         //         // 注意这里并没有销毁该连接，该连接仍然可用，但需要重新获取
         //         conn.release();
@@ -71,16 +65,18 @@ async function query(sql, params) {
     });
 }
 
-function close() {
+async function close() {
+    await new Promise((resolve, reject) => {
         if (!pool) return;
         pool.end(function (err) {
             console.log('关闭连接池', err);
+            resolve(err);
+        });
     });
-    pool = null;
 }
 
 module.exports = {
-    createPool: createPool,
+    create: create,
     query: query,
     close: close,
 }

utils/dbUtils.js

@@ -4,16 +4,16 @@ const fs = require('fs');
 const path = require('path');
 
 let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8'));
-let database = null;
+let databaseName = null;
 
-function create(databaseName) {
+function create({ database }) {
-    database = databaseName;
+    databaseName = database;
 }
 
 async function query(sql, params) {
     let config = {
         ...globalConfig.mysql,
-        database: database,
+        database: databaseName,
     };
     // console.log(config);
 
@@ -34,6 +34,8 @@ async function query(sql, params) {
 }
 
 module.exports = {
+    setConnectionLimit: () => { },
     create: create,
     query: query,
+    close: () => { },
 }