coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

添加 assistant 助手；其他调整（大调整）

Browse Source
This commit is contained in:
小墨
2022-10-29 00:16:36 +08:00
parent ddde1b28f9
commit ce20720c60
21 changed files with 973 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
netease_music/src/getInfo/albumInfoUtils.js
View File

@@ -140,17 +140,26 @@ async function fetch({ albumId, debug = false, update = false }) {
};
// console.log("albumInfo", albumInfo);
// 插入待爬取表
await dataManager.wait_check.insert("song", songIds);
await dataManager.wait_check.insert("lyric", songIds);
await dataManager.wait_check.insert("comment", songIds);
// 插入关联关系
if (albumId > 0) {
let songAlbumRel = songIds.map(songId => [songId, albumId]);
await dataManager.song_album.insertCollection(songAlbumRel);
}
// 插入数据
if (update) {
await dataManager.album.update(albumId, albumInfo);
} else {
await dataManager.album.insert(albumInfo);
}
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("album", [albumId]);
}
module.exports = {

9
netease_music/src/getInfo/artistInfoUtils.js
View File

@@ -96,13 +96,22 @@ async function fetch({ artistId, debug = false }) {
};
// console.log("artistInfo", artistInfo);
// 插入待爬取表
await dataManager.wait_check.insert("song", songIds);
await dataManager.wait_check.insert("lyric", songIds);
await dataManager.wait_check.insert("comment", songIds);
// 插入关联关系
if (artistId > 0) {
let songArtistRel = songIds.map(songId => [songId, artistId]);
await dataManager.song_artist.insertCollection(songArtistRel);
}
// 插入数据
await dataManager.artist.insert(artistInfo);
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("artist", [artistId]);
}
module.exports = {

27
netease_music/src/getInfo/commentUtils.js
View File

@@ -14,27 +14,12 @@ const { comment_music } = require('NeteaseCloudMusicApi');
async function fetchAll({ args = {} }) {
console.log("start fetching comment ...");
// 首先将需要爬取的song_id导入comment_progress表
await dbUtils.query(`
INSERT IGNORE INTO comment_progress ( song_id )
SELECT DISTINCT song_id FROM song WHERE song_id NOT IN ( SELECT song_id FROM comment_progress )
`, []);
let whereClause = [
args.min ? `song_id > ${args.min}` : '1=1',
args.max ? `song_id <= ${args.max}` : '1=1',
].join(' AND ');
var sql = `
SELECT song_id FROM comment_progress WHERE ${whereClause} AND current_status != 2
ORDER BY current_status DESC${args.order ? `, song_id ${args.order}` : ''}
${args.limit ? `LIMIT ${args.limit}` : ''}
`;
console.log(sql);
// 首先查询有无正在爬取中的记录
var songIds = await dbUtils.query(sql, []);
songIds = songIds.map(item => item.song_id);
// // 首先将需要爬取的song_id导入comment_progress表
// await dbUtils.query(`
// INSERT IGNORE INTO comment_progress ( song_id )
// SELECT song_id FROM wait_fetch_comment WHERE song_id NOT IN ( SELECT song_id FROM comment_progress )
// `, []);
let songIds = await dataManager.comment.getIdsToFetch(args);
for (let i = 0; i < songIds.length; i++) {
await global.checkIsExit();
const songId = songIds[i];

19
netease_music/src/getInfo/lyricInfoUtils.js
View File

@@ -10,19 +10,7 @@ const dbUtils = global.dbUtils;
// 从数据库中查出还缺少的歌词，并进行爬取
async function fetchAll({ args = {} }) {
console.log("start fetching lyrics ...");
let whereClause = [
args.min ? `song_id > ${args.min}` : '1=1',
args.max ? `song_id <= ${args.max}` : '1=1',
].join(' AND ');
var sql = `
SELECT song_id FROM song WHERE ${whereClause} AND song_id NOT IN ( SELECT song_id FROM lyric )
${args.order ? `ORDER BY song_id ${args.order}` : ''}
${args.limit ? `LIMIT ${args.limit}` : ''}
`;
console.log(sql);
var songIds = await dbUtils.query(sql, []);
songIds = songIds.map(song => song.song_id);
let songIds = await dataManager.lyric.getIdsToFetch(args);
for (let i = 0; i < songIds.length; i++) {
await global.checkIsExit();
const songId = songIds[i];
@@ -78,7 +66,12 @@ async function fetch({ songId, debug = false }) {
version: lyric.version,
};
// console.log("lyricInfo", lyricInfo);
// 插入数据
await dataManager.lyric.insert(lyricInfo);
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("lyric", [songId]);
}
module.exports = {

8
netease_music/src/getInfo/songInfoUtils.js
View File

@@ -85,11 +85,19 @@ async function fetch({ songIdArray, debug = false }) {
if (songInfoList.length == 0) return;
console.log("插入数据库");
// 插入待爬取表
await dataManager.wait_check.insert("album", albumIds);
await dataManager.wait_check.insert("artist", artistIds);
// 插入关联关系
await dataManager.song_album.insertCollection(songAlbumRel);
await dataManager.song_artist.insertCollection(songArtistRel);
// 插入数据
await dataManager.song.insertCollection(songInfoList); // image 因为接口没有返回，所以不更新
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("song", [songId]);
}
// 获取音乐详情