coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

使用连接池;退出检查优化

Browse Source
This commit is contained in:
小墨
2022-10-01 19:45:49 +08:00
parent 074cbc124f
commit cf4449604d
7 changed files with 92 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
netease_music/index.js
View File

@@ -1,21 +1,55 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../utils/dbUtils');
// const dbUtils = require('../utils/dbUtils');
const dbUtils = require('../utils/dbPoolUtils');
const requestUtils = require('../utils/requestUtils');
const sleepUtils = require('../utils/sleepUtils');
dbUtils.create({
database: "neteaseMusic", // 指定数据库
connectionLimit: 8, // 设置数据库连接池数量
});
global.dbUtils = dbUtils;
const songInfoUtils = require('./src/getInfo/songInfoUtils');
const artistInfoUtils = require('./src/getInfo/artistInfoUtils');
const albumInfoUtils = require('./src/getInfo/albumInfoUtils');
// 退出检查
async function checkIsExit() {
if (fs.readFileSync('stop.txt') != "1")
return;
console.log();
console.log(`收到退出指令，准备退出...`);
await sleepUtils.sleep(500);
await dbUtils.close();
console.log(`数据库连接池已关闭`);
await sleepUtils.sleep(100);
process.exit(0);
}
async function test() {
console.log("neteaseMusic test...");
let songIds = (await dbUtils.query(`
SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
`, [])).map(song => song.song_id);
for (let songId of songIds) {
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
var json = await requestUtils.getApiResult(url);
var lyric = JSON.parse(json).lrc;
console.log(lyric);
}
// fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
}
async function main() {
console.log("neteaseMusic Starting...");
// 指定数据库
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
console.log(`数据统计: ${await statistics()}`);
// getMusicInfo({ songId: "1855221507" });
// getArtistInfo({ artistId: "1079074" });
@@ -28,7 +62,7 @@ async function main() {
// 删除脏数据
var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []);
var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
console.log(affectRows1.affectedRows, affectRows2.affectedRows);
console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows);
await startGet(1);
await sleepUtils.sleep(2000);
@@ -36,7 +70,6 @@ async function main() {
}
async function startGet(sleepTime) {
// 从数据库中查出还缺少的歌曲，并进行爬取
console.log("start fetching songs ...");
let songIds = await dbUtils.query(`
@@ -46,6 +79,7 @@ async function startGet(sleepTime) {
`, []);
songIds = songIds.map(item => item.song_id);
for (let i = 0; i < songIds.length; i++) {
await checkIsExit();
const songId = songIds[i];
console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`);
try {
@@ -54,9 +88,6 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
// 从数据库中查出还缺少的专辑，并进行爬取
@@ -66,6 +97,7 @@ async function startGet(sleepTime) {
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
await checkIsExit();
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try {
@@ -74,9 +106,6 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
// 从数据库中查出还缺少的歌手，并进行爬取
@@ -86,6 +115,7 @@ async function startGet(sleepTime) {
`, []);
artistIds = artistIds.map(item => item.artist_id);
for (let i = 0; i < artistIds.length; i++) {
await checkIsExit();
const artistId = artistIds[i];
console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`);
try {
@@ -94,16 +124,12 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
}
async function update() {
console.log("neteaseMusic update ...");
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
console.log(`数据统计: ${await statistics()}`);
let sleepTime = 100;
@@ -114,6 +140,7 @@ async function update() {
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
await checkIsExit();
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try {
@@ -122,9 +149,6 @@ async function update() {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
}
@@ -133,28 +157,25 @@ async function statistics() {
SELECT
song_count,
album_count,
album_v3_count,
album_v4_count,
album_v1_count,
artist_count,
song_album_count,
song_artist_count
FROM
( SELECT count(*) AS song_count FROM song ) t1,
( SELECT count(*) AS album_count FROM album ) t2,
( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1,
( SELECT count(*) AS artist_count FROM artist ) t3,
( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
let result = await dbUtils.query(sql, []);
let songCount = result[0].song_count;
let albumCount = result[0].album_count;
let albumV3Count = result[0].album_v3_count;
let albumV4Count = result[0].album_v4_count;
let albumV1Count = result[0].album_v1_count;
let artistCount = result[0].artist_count;
let songAlbumCount = result[0].song_album_count;
let songArtistCount = result[0].song_artist_count;
return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
}
module.exports = {

5
netease_music/src/getInfo/albumInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取专辑详情
async function fetch({ albumId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
@@ -95,7 +96,7 @@ async function fetch({ albumId }) {
image: albumInfo.image,
pub_date: albumInfo.pubDate,
company: albumInfo.company,
version: 4
version: 1
});
songIds.forEach(function (songId) {
if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0)

3
netease_music/src/getInfo/artistInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取音乐人详情
async function fetch({ artistId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);

3
netease_music/src/getInfo/songInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取音乐详情
async function fetch({ songId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]);

3
netease_music/src/getInfo/userInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取用户详情
async function fetch({ userId }) {
let url = `https://music.163.com/user/home?id=${userId}`;

60
utils/dbPoolUtils.js
View File

@@ -5,9 +5,9 @@ const path = require('path');
let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8'));
let pool = null;
function createPool(database) {
function create({ database, connectionLimit = 10 }) {
let config = {
connectionLimit: 10, //连接数量，默认是10
connectionLimit: connectionLimit, //连接数量，默认是10
...globalConfig.mysql,
database: database,
};
@@ -21,10 +21,10 @@ function createPool(database) {
// console.log('获取连接', conn.threadId);
// });
// //在连接池中建立新连接时，将触发该事件
// pool.on('connection', function (conn) {
// console.log('建立新连接', conn.threadId);
// });
//在连接池中建立新连接时，将触发该事件
pool.on('connection', function (conn) {
console.log('建立新连接', conn.threadId);
});
// //等待可用连接时，将触发该事件
// pool.on('enqueue', function () {
@@ -38,49 +38,45 @@ function createPool(database) {
}
async function query(sql, params) {
if (!pool) {
console.error('Database connection pool is not initialized yet.');
return;
}
return await new Promise(function (resolve, reject) {
//pool.query()方法可以自动的帮我们在连接池中获取可用连接
if (!pool) {
console.error('Database connection pool is not initialized yet.');
resolve(data);
return;
}
// pool.query()方法可以自动的帮我们在连接池中获取可用连接
pool.query(sql, params, function (err, data) {
if (err) reject(err);
// console.log(data);
resolve(data);
});
// //当然我们也可以手动获取可用连接
// // 当然我们也可以手动获取可用连接
// pool.getConnection(function (err, conn) {
// if (err) {
// throw err;
// }
// conn.query('select * from `order`', function (err, data) {
// if (err) {
// throw err;
// }
// data.forEach(function (value) {
// console.log(value.id, value.order_id, value.user_id);
// });
// //连接用完之后，需要释放，重新放回连接池中。
// //注意这里并没有销毁该连接，该连接仍然可用，但需要重新获取
// if (err) reject(err);
// conn.query(sql, function (err, data) {
// if (err) reject(err);
// resolve(data);
// // 连接用完之后，需要释放，重新放回连接池中。
// // 注意这里并没有销毁该连接，该连接仍然可用，但需要重新获取
// conn.release();
// });
// });
});
}
function close() {
if (!pool) return;
pool.end(function (err) {
console.log('关闭连接池', err);
async function close() {
await new Promise((resolve, reject) => {
if (!pool) return;
pool.end(function (err) {
console.log('关闭连接池', err);
resolve(err);
});
});
pool = null;
}
module.exports = {
createPool: createPool,
create: create,
query: query,
close: close,
}

10
utils/dbUtils.js
View File

@@ -4,16 +4,16 @@ const fs = require('fs');
const path = require('path');
let globalConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '../config.json'), 'utf8'));
let database = null;
let databaseName = null;
function create(databaseName) {
database = databaseName;
function create({ database }) {
databaseName = database;
}
async function query(sql, params) {
let config = {
...globalConfig.mysql,
database: database,
database: databaseName,
};
// console.log(config);
@@ -34,6 +34,8 @@ async function query(sql, params) {
}
module.exports = {
setConnectionLimit: () => { },
create: create,
query: query,
close: () => { },
}