coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

使用连接池;退出检查优化

Browse Source
This commit is contained in:
小墨
2022-10-01 19:45:49 +08:00
parent 074cbc124f
commit cf4449604d
7 changed files with 92 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
netease_music/index.js
View File

@@ -1,21 +1,55 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../utils/dbUtils');
// const dbUtils = require('../utils/dbUtils');
const dbUtils = require('../utils/dbPoolUtils');
const requestUtils = require('../utils/requestUtils');
const sleepUtils = require('../utils/sleepUtils');
dbUtils.create({
database: "neteaseMusic", // 指定数据库
connectionLimit: 8, // 设置数据库连接池数量
});
global.dbUtils = dbUtils;
const songInfoUtils = require('./src/getInfo/songInfoUtils');
const artistInfoUtils = require('./src/getInfo/artistInfoUtils');
const albumInfoUtils = require('./src/getInfo/albumInfoUtils');
// 退出检查
async function checkIsExit() {
if (fs.readFileSync('stop.txt') != "1")
return;
console.log();
console.log(`收到退出指令，准备退出...`);
await sleepUtils.sleep(500);
await dbUtils.close();
console.log(`数据库连接池已关闭`);
await sleepUtils.sleep(100);
process.exit(0);
}
async function test() {
console.log("neteaseMusic test...");
let songIds = (await dbUtils.query(`
SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
`, [])).map(song => song.song_id);
for (let songId of songIds) {
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
var json = await requestUtils.getApiResult(url);
var lyric = JSON.parse(json).lrc;
console.log(lyric);
}
// fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
}
async function main() {
console.log("neteaseMusic Starting...");
// 指定数据库
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
console.log(`数据统计: ${await statistics()}`);
// getMusicInfo({ songId: "1855221507" });
// getArtistInfo({ artistId: "1079074" });
@@ -28,7 +62,7 @@ async function main() {
// 删除脏数据
var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []);
var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
console.log(affectRows1.affectedRows, affectRows2.affectedRows);
console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows);
await startGet(1);
await sleepUtils.sleep(2000);
@@ -36,7 +70,6 @@ async function main() {
}
async function startGet(sleepTime) {
// 从数据库中查出还缺少的歌曲，并进行爬取
console.log("start fetching songs ...");
let songIds = await dbUtils.query(`
@@ -46,6 +79,7 @@ async function startGet(sleepTime) {
`, []);
songIds = songIds.map(item => item.song_id);
for (let i = 0; i < songIds.length; i++) {
await checkIsExit();
const songId = songIds[i];
console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`);
try {
@@ -54,9 +88,6 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
// 从数据库中查出还缺少的专辑，并进行爬取
@@ -66,6 +97,7 @@ async function startGet(sleepTime) {
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
await checkIsExit();
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try {
@@ -74,9 +106,6 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
// 从数据库中查出还缺少的歌手，并进行爬取
@@ -86,6 +115,7 @@ async function startGet(sleepTime) {
`, []);
artistIds = artistIds.map(item => item.artist_id);
for (let i = 0; i < artistIds.length; i++) {
await checkIsExit();
const artistId = artistIds[i];
console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`);
try {
@@ -94,16 +124,12 @@ async function startGet(sleepTime) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
}
async function update() {
console.log("neteaseMusic update ...");
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
console.log(`数据统计: ${await statistics()}`);
let sleepTime = 100;
@@ -114,6 +140,7 @@ async function update() {
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
await checkIsExit();
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try {
@@ -122,9 +149,6 @@ async function update() {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
}
@@ -133,28 +157,25 @@ async function statistics() {
SELECT
song_count,
album_count,
album_v3_count,
album_v4_count,
album_v1_count,
artist_count,
song_album_count,
song_artist_count
FROM
( SELECT count(*) AS song_count FROM song ) t1,
( SELECT count(*) AS album_count FROM album ) t2,
( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1,
( SELECT count(*) AS artist_count FROM artist ) t3,
( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
let result = await dbUtils.query(sql, []);
let songCount = result[0].song_count;
let albumCount = result[0].album_count;
let albumV3Count = result[0].album_v3_count;
let albumV4Count = result[0].album_v4_count;
let albumV1Count = result[0].album_v1_count;
let artistCount = result[0].artist_count;
let songAlbumCount = result[0].song_album_count;
let songArtistCount = result[0].song_artist_count;
return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
}
module.exports = {

5
netease_music/src/getInfo/albumInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取专辑详情
async function fetch({ albumId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
@@ -95,7 +96,7 @@ async function fetch({ albumId }) {
image: albumInfo.image,
pub_date: albumInfo.pubDate,
company: albumInfo.company,
version: 4
version: 1
});
songIds.forEach(function (songId) {
if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0)

3
netease_music/src/getInfo/artistInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取音乐人详情
async function fetch({ artistId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);

3
netease_music/src/getInfo/songInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取音乐详情
async function fetch({ songId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]);

3
netease_music/src/getInfo/userInfoUtils.js
View File

@@ -1,10 +1,11 @@
const fs = require('fs');
const path = require('path');
const dbUtils = require('../../../utils/dbUtils');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 获取用户详情
async function fetch({ userId }) {
let url = `https://music.163.com/user/home?id=${userId}`;