使用连接池;退出检查优化
This commit is contained in:
@@ -1,21 +1,55 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const dbUtils = require('../utils/dbUtils');
|
||||
// const dbUtils = require('../utils/dbUtils');
|
||||
const dbUtils = require('../utils/dbPoolUtils');
|
||||
|
||||
const requestUtils = require('../utils/requestUtils');
|
||||
const sleepUtils = require('../utils/sleepUtils');
|
||||
|
||||
dbUtils.create({
|
||||
database: "neteaseMusic", // 指定数据库
|
||||
connectionLimit: 8, // 设置数据库连接池数量
|
||||
});
|
||||
|
||||
global.dbUtils = dbUtils;
|
||||
const songInfoUtils = require('./src/getInfo/songInfoUtils');
|
||||
const artistInfoUtils = require('./src/getInfo/artistInfoUtils');
|
||||
const albumInfoUtils = require('./src/getInfo/albumInfoUtils');
|
||||
|
||||
|
||||
// 退出检查
|
||||
async function checkIsExit() {
|
||||
if (fs.readFileSync('stop.txt') != "1")
|
||||
return;
|
||||
console.log();
|
||||
console.log(`收到退出指令，准备退出...`);
|
||||
await sleepUtils.sleep(500);
|
||||
await dbUtils.close();
|
||||
console.log(`数据库连接池已关闭`);
|
||||
await sleepUtils.sleep(100);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
async function test() {
|
||||
console.log("neteaseMusic test...");
|
||||
|
||||
let songIds = (await dbUtils.query(`
|
||||
SELECT DISTINCT song_id FROM song_lyric WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
|
||||
`, [])).map(song => song.song_id);
|
||||
|
||||
for (let songId of songIds) {
|
||||
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
|
||||
var json = await requestUtils.getApiResult(url);
|
||||
var lyric = JSON.parse(json).lrc;
|
||||
console.log(lyric);
|
||||
}
|
||||
// fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
|
||||
}
|
||||
|
||||
|
||||
async function main() {
|
||||
console.log("neteaseMusic Starting...");
|
||||
|
||||
// 指定数据库
|
||||
dbUtils.create("neteaseMusic");
|
||||
console.log(`${await statistics()}`);
|
||||
console.log(`数据统计: ${await statistics()}`);
|
||||
|
||||
// getMusicInfo({ songId: "1855221507" });
|
||||
// getArtistInfo({ artistId: "1079074" });
|
||||
@@ -28,7 +62,7 @@ async function main() {
|
||||
// 删除脏数据
|
||||
var affectRows1 = await dbUtils.query(`DELETE FROM song_artist_relation WHERE song_id = 0 OR artist_id = 0`, []);
|
||||
var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
|
||||
console.log(affectRows1.affectedRows, affectRows2.affectedRows);
|
||||
console.log(`删除脏数据 affectRows:`, affectRows1.affectedRows, affectRows2.affectedRows);
|
||||
|
||||
await startGet(1);
|
||||
await sleepUtils.sleep(2000);
|
||||
@@ -36,7 +70,6 @@ async function main() {
|
||||
}
|
||||
|
||||
async function startGet(sleepTime) {
|
||||
|
||||
// 从数据库中查出还缺少的歌曲，并进行爬取
|
||||
console.log("start fetching songs ...");
|
||||
let songIds = await dbUtils.query(`
|
||||
@@ -46,6 +79,7 @@ async function startGet(sleepTime) {
|
||||
`, []);
|
||||
songIds = songIds.map(item => item.song_id);
|
||||
for (let i = 0; i < songIds.length; i++) {
|
||||
await checkIsExit();
|
||||
const songId = songIds[i];
|
||||
console.log(`${i}/${songIds.length} | song: ${songId} | ${await statistics()}`);
|
||||
try {
|
||||
@@ -54,9 +88,6 @@ async function startGet(sleepTime) {
|
||||
console.error(err);
|
||||
}
|
||||
await sleepUtils.sleep(sleepTime);
|
||||
if (fs.readFileSync('stop.txt') == "1") {
|
||||
throw new Error(`Stopped`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从数据库中查出还缺少的专辑，并进行爬取
|
||||
@@ -66,6 +97,7 @@ async function startGet(sleepTime) {
|
||||
`, []);
|
||||
albumIds = albumIds.map(item => item.album_id);
|
||||
for (let i = 0; i < albumIds.length; i++) {
|
||||
await checkIsExit();
|
||||
const albumId = albumIds[i];
|
||||
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
|
||||
try {
|
||||
@@ -74,9 +106,6 @@ async function startGet(sleepTime) {
|
||||
console.error(err);
|
||||
}
|
||||
await sleepUtils.sleep(sleepTime);
|
||||
if (fs.readFileSync('stop.txt') == "1") {
|
||||
throw new Error(`Stopped`);
|
||||
}
|
||||
}
|
||||
|
||||
// 从数据库中查出还缺少的歌手，并进行爬取
|
||||
@@ -86,6 +115,7 @@ async function startGet(sleepTime) {
|
||||
`, []);
|
||||
artistIds = artistIds.map(item => item.artist_id);
|
||||
for (let i = 0; i < artistIds.length; i++) {
|
||||
await checkIsExit();
|
||||
const artistId = artistIds[i];
|
||||
console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await statistics()}`);
|
||||
try {
|
||||
@@ -94,16 +124,12 @@ async function startGet(sleepTime) {
|
||||
console.error(err);
|
||||
}
|
||||
await sleepUtils.sleep(sleepTime);
|
||||
if (fs.readFileSync('stop.txt') == "1") {
|
||||
throw new Error(`Stopped`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function update() {
|
||||
console.log("neteaseMusic update ...");
|
||||
dbUtils.create("neteaseMusic");
|
||||
console.log(`${await statistics()}`);
|
||||
console.log(`数据统计: ${await statistics()}`);
|
||||
|
||||
let sleepTime = 100;
|
||||
|
||||
@@ -114,6 +140,7 @@ async function update() {
|
||||
`, []);
|
||||
albumIds = albumIds.map(item => item.album_id);
|
||||
for (let i = 0; i < albumIds.length; i++) {
|
||||
await checkIsExit();
|
||||
const albumId = albumIds[i];
|
||||
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
|
||||
try {
|
||||
@@ -122,9 +149,6 @@ async function update() {
|
||||
console.error(err);
|
||||
}
|
||||
await sleepUtils.sleep(sleepTime);
|
||||
if (fs.readFileSync('stop.txt') == "1") {
|
||||
throw new Error(`Stopped`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,28 +157,25 @@ async function statistics() {
|
||||
SELECT
|
||||
song_count,
|
||||
album_count,
|
||||
album_v3_count,
|
||||
album_v4_count,
|
||||
album_v1_count,
|
||||
artist_count,
|
||||
song_album_count,
|
||||
song_artist_count
|
||||
FROM
|
||||
( SELECT count(*) AS song_count FROM song ) t1,
|
||||
( SELECT count(*) AS album_count FROM album ) t2,
|
||||
( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
|
||||
( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
|
||||
( SELECT count(*) AS album_v1_count FROM album WHERE version = 1 ) t3_1,
|
||||
( SELECT count(*) AS artist_count FROM artist ) t3,
|
||||
( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
|
||||
( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
|
||||
let result = await dbUtils.query(sql, []);
|
||||
let songCount = result[0].song_count;
|
||||
let albumCount = result[0].album_count;
|
||||
let albumV3Count = result[0].album_v3_count;
|
||||
let albumV4Count = result[0].album_v4_count;
|
||||
let albumV1Count = result[0].album_v1_count;
|
||||
let artistCount = result[0].artist_count;
|
||||
let songAlbumCount = result[0].song_album_count;
|
||||
let songArtistCount = result[0].song_artist_count;
|
||||
return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
|
||||
return `song: ${songCount}, album: ${albumCount}(v1: ${albumV1Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
Reference in New Issue
Block a user