coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

bugfix；添加auto.bat

Browse Source
This commit is contained in:
小墨
2022-10-29 01:21:48 +08:00
parent ce20720c60
commit d3245038d7
11 changed files with 77 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
netease_music/src/assistantUtils.js
View File

@@ -12,13 +12,14 @@ function getDiffSet(a, b) {
async function migrateIdsFromCheckToFetch(tableName, fieldName, insertSql = null) {
console.log(`更新待爬取列表: ${tableName}`);
let stepLength = 1000;
let stepLength = 5000;
while (true) {
// 从 check 表中分块查出待处理数据
let idsResult = await dbUtils.query(`SELECT id FROM wait_check_${tableName} LIMIT ${stepLength}`, []);
let ids = idsResult.map(row => row.id);
// console.log("ids", ids);
if (ids.length == 0) {
console.log(`${tableName} done.`);
break;
};
@@ -39,19 +40,25 @@ async function migrateIdsFromCheckToFetch(tableName, fieldName, insertSql = null
// 从待检查表中删除
if (ids.length > 0)
await dbUtils.query(`DELETE FROM wait_check_${tableName} WHERE id IN ?`, [[ids]]);
console.log(`table: ${tableName} | ${ids[0]} - ${ids.slice(-1)[0]}`)
console.log(`table: ${tableName} | ${ids[0]} - ${ids.slice(-1)[0]}`);
}
}
function getPromise(tableName, fieldName, insertSql) {
return new Promise(async function (resolve) {
await migrateIdsFromCheckToFetch(tableName, fieldName, insertSql);
resolve();
});
}
async function updateWaitTable() {
await migrateIdsFromCheckToFetch("song", "song_id");
await migrateIdsFromCheckToFetch("lyric", "song_id");
await migrateIdsFromCheckToFetch("comment", "song_id", `INSERT IGNORE INTO comment_progress (song_id) VALUES ?`);
await migrateIdsFromCheckToFetch("album", "album_id");
await migrateIdsFromCheckToFetch("artist", "artist_id");
// comment 搬到 comment_progress
console.log("done.\n");
await Promise.all([
getPromise("song", "song_id"),
getPromise("lyric", "song_id"),
getPromise("comment", "song_id", `INSERT IGNORE INTO comment_progress (song_id) VALUES ?`),
getPromise("album", "album_id"),
getPromise("artist", "artist_id")
]);
console.log("All done.\n");
}
module.exports = {

31
netease_music/src/dataManager.js
View File

@@ -60,18 +60,18 @@ module.exports = {
sql = `SELECT album_id FROM album WHERE (full_description = '' or full_description is null) and description like '%专辑《%》，简介：%' and description not regexp '^.*?专辑《.*?》，简介：[:space:]*?。，更多.*$'`;
} else {
let whereClause = [
args.min ? `album_id > ${args.min}` : '1=1',
args.max ? `album_id <= ${args.max}` : '1=1',
args.min ? `id > ${args.min}` : '1=1',
args.max ? `id <= ${args.max}` : '1=1',
].join(' AND ');
sql = `
SELECT album_id FROM wait_fetch_album WHERE ${whereClause}
${args.order ? `ORDER BY album_id ${args.order}` : ''}
SELECT id FROM wait_fetch_album WHERE ${whereClause}
${args.order ? `ORDER BY id ${args.order}` : ''}
${args.limit ? `LIMIT ${args.limit}` : ''}
`;
}
console.log(sql);
let albumIds = await dbUtils.query(sql, []);
albumIds = albumIds.map(item => item.album_id);
albumIds = albumIds.map(item => item.id);
return albumIds;
},
},
@@ -84,17 +84,17 @@ module.exports = {
getIdsToFetch: async (args) => {
let whereClause = [
args.min ? `artist_id > ${args.min}` : '1=1',
args.max ? `artist_id <= ${args.max}` : '1=1',
args.min ? `id > ${args.min}` : '1=1',
args.max ? `id <= ${args.max}` : '1=1',
].join(' AND ');
let sql = `
SELECT artist_id FROM wait_fetch_artist WHERE ${whereClause}
${args.order ? `ORDER BY artist_id ${args.order}` : ''}
SELECT id FROM wait_fetch_artist WHERE ${whereClause}
${args.order ? `ORDER BY id ${args.order}` : ''}
${args.limit ? `LIMIT ${args.limit}` : ''}
`;
console.log(sql);
let artistIds = await dbUtils.query(sql, []);
artistIds = artistIds.map(item => item.artist_id);
artistIds = artistIds.map(item => item.id);
return artistIds;
},
},
@@ -107,17 +107,17 @@ module.exports = {
getIdsToFetch: async (args) => {
let whereClause = [
args.min ? `song_id > ${args.min}` : '1=1',
args.max ? `song_id <= ${args.max}` : '1=1',
args.min ? `id > ${args.min}` : '1=1',
args.max ? `id <= ${args.max}` : '1=1',
].join(' AND ');
var sql = `
SELECT song_id FROM wait_fetch_lyric WHERE ${whereClause}
${args.order ? `ORDER BY song_id ${args.order}` : ''}
SELECT id FROM wait_fetch_lyric WHERE ${whereClause}
${args.order ? `ORDER BY id ${args.order}` : ''}
${args.limit ? `LIMIT ${args.limit}` : ''}
`;
console.log(sql);
let songIds = await dbUtils.query(sql, []);
songIds = songIds.map(song => song.song_id);
songIds = songIds.map(item => item.id);
return songIds;
},
},
@@ -218,6 +218,7 @@ module.exports = {
wait_fetch: {
deleteCollection: async function (type, ids) {
// console.log("wait_fetch.deleteCollection", type, ids);
if (ids.length > 0)
return await dbUtils.query(`DELETE FROM wait_fetch_${type} WHERE id IN ?`, [[ids]]);
}

2
netease_music/src/getInfo/albumInfoUtils.js
View File

@@ -48,6 +48,8 @@ async function fetch({ albumId, debug = false, update = false }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
if (!debug && !update && result[0].count > 0) {
console.log(`数据库中已有数据，跳过 albumId: ${albumId}`);
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("album", [albumId]);
return;
} else if (update && result[0].count == 0) {
console.log(`数据库中沒有数据，跳过 albumId: ${albumId}`);

2
netease_music/src/getInfo/artistInfoUtils.js
View File

@@ -44,6 +44,8 @@ async function fetch({ artistId, debug = false }) {
let result = await dbUtils.query('SELECT count(*) as count FROM artist WHERE artist_id = ?', [artistId]);
if (result[0].count > 0 && !debug) {
console.log(`数据库中已有数据，跳过 artistId: ${artistId}`);
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("artist", [artistId]);
return;
}

2
netease_music/src/getInfo/lyricInfoUtils.js
View File

@@ -30,6 +30,8 @@ async function fetch({ songId, debug = false }) {
if (result[0].count > 0 && !debug) {
// 这里暂时跳过，后期可能要考虑歌词version更新的问题
console.log(`数据库中已有数据，跳过 songId: ${songId}`);
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("lyric", [songId]);
return;
}

2
netease_music/src/getInfo/songInfoUtils.js
View File

@@ -97,7 +97,7 @@ async function fetch({ songIdArray, debug = false }) {
await dataManager.song.insertCollection(songInfoList); // image 因为接口没有返回，所以不更新
// 从待爬取表中删除记录
await dataManager.wait_fetch.deleteCollection("song", [songId]);
await dataManager.wait_fetch.deleteCollection("song", songIdArray);
}
// 获取音乐详情