coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

fetchAll从index.js提到各个utils中

Browse Source
This commit is contained in:
小墨
2022-10-01 22:00:29 +08:00
parent 5d2bfccb4b
commit 3dcb71b5a3
6 changed files with 196 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
netease_music/src/getInfo/albumInfoUtils.js
View File

@@ -2,6 +2,7 @@ const fs = require('fs');
const path = require('path');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
@@ -20,6 +21,26 @@ async function getFromDatabase({ albumId }) {
return albumInfo;
}
// 从数据库中查出还缺少的专辑，并进行爬取
async function fetchAll() {
console.log("start fetching albums ...")
var albumIds = await dbUtils.query(`
SELECT DISTINCT album_id FROM song_album_relation WHERE album_id NOT IN ( SELECT DISTINCT album_id FROM album )
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
await global.checkIsExit();
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await global.statistics()}`);
try {
await fetch({ albumId: albumId });
} catch (err) {
console.error(err);
}
await sleepUtils.sleep(global.sleepTime);
}
}
// 获取专辑详情
async function fetch({ albumId, debug = false }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
@@ -186,5 +207,6 @@ async function update({ albumId }) {
module.exports = {
getFromDatabase: getFromDatabase,
fetch: fetch,
fetchAll: fetchAll,
update: update,
}

22
netease_music/src/getInfo/artistInfoUtils.js
View File

@@ -2,6 +2,7 @@ const fs = require('fs');
const path = require('path');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
@@ -20,6 +21,26 @@ async function getFromDatabase({ artistId }) {
return artistInfo;
}
// 从数据库中查出还缺少的歌手，并进行爬取
async function fetchAll() {
console.log("start fetching artists ...")
var artistIds = await dbUtils.query(`
SELECT DISTINCT artist_id FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist )
`, []);
artistIds = artistIds.map(item => item.artist_id);
for (let i = 0; i < artistIds.length; i++) {
await global.checkIsExit();
const artistId = artistIds[i];
console.log(`${i}/${artistIds.length} | artist: ${artistId} | ${await global.statistics()}`);
try {
await fetch({ artistId: artistId });
} catch (err) {
console.error(err);
}
await sleepUtils.sleep(global.sleepTime);
}
}
// 获取音乐人详情
async function fetch({ artistId, debug = false }) {
let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);
@@ -85,4 +106,5 @@ async function fetch({ artistId, debug = false }) {
module.exports = {
getFromDatabase: getFromDatabase,
fetch: fetch,
fetchAll: fetchAll,
}

24
netease_music/src/getInfo/lyricInfoUtils.js
View File

@@ -2,11 +2,32 @@ const fs = require('fs');
const path = require('path');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
// 从数据库中查出还缺少的歌词，并进行爬取
async function fetchAll() {
console.log("start fetching lyrics ...");
var songIds = await dbUtils.query(`
SELECT DISTINCT song_id FROM song WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM lyric )
`, []);
songIds = songIds.map(song => song.song_id);
for (let i = 0; i < songIds.length; i++) {
await global.checkIsExit();
const songId = songIds[i];
console.log(`${i}/${songIds.length} | lyric: ${songId} | ${await global.statistics()}`);
try {
await fetch({ songId: songId });
} catch (err) {
console.error(err);
}
await sleepUtils.sleep(global.sleepTime);
}
}
// 获取歌词详情
async function fetch({ songId }) {
async function fetch({ songId, debug = false }) {
var url = `https://music.163.com/api/song/lyric?id=${songId}&lv=1`;
try {
@@ -41,4 +62,5 @@ async function fetch({ songId }) {
module.exports = {
fetch: fetch,
fetchAll: fetchAll,
}

24
netease_music/src/getInfo/songInfoUtils.js
View File

@@ -2,6 +2,7 @@ const fs = require('fs');
const path = require('path');
const requestUtils = require('../../../utils/requestUtils');
const sleepUtils = require('../../../utils/sleepUtils');
const dbUtils = global.dbUtils;
@@ -22,6 +23,28 @@ async function getFromDatabase({ songId }) {
return songInfo;
}
// 从数据库中查出还缺少的歌曲，并进行爬取
async function fetchAll() {
console.log("start fetching songs ...");
var songIds = await dbUtils.query(`
SELECT DISTINCT song_id FROM song_artist_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
UNION
SELECT DISTINCT song_id FROM song_album_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song )
`, []);
songIds = songIds.map(item => item.song_id);
for (let i = 0; i < songIds.length; i++) {
await global.checkIsExit();
const songId = songIds[i];
console.log(`${i}/${songIds.length} | song: ${songId} | ${await global.statistics()}`);
try {
await fetch({ songId: songId });
} catch (err) {
console.error(err);
}
await sleepUtils.sleep(global.sleepTime);
}
}
// 获取音乐详情
async function fetch({ songId, debug = false }) {
let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]);
@@ -119,4 +142,5 @@ async function fetch({ songId, debug = false }) {
module.exports = {
getFromDatabase: getFromDatabase,
fetch: fetch,
fetchAll: fetchAll,
}