插入关联表时同事插入wait_check表；统一查询将要爬取的id代码到dataManager.js
This commit is contained in:
@@ -20,8 +20,27 @@ module.exports = {
|
||||
songInfo.noCopyrightRcmd, songInfo.mv, songInfo.single, songInfo.version, 2
|
||||
])]);
|
||||
},
|
||||
|
||||
getIdsToFetch: async (args) => {
|
||||
let whereClause = [
|
||||
args.min ? `song_id > ${args.min}` : '1=1',
|
||||
args.max ? `song_id <= ${args.max}` : '1=1',
|
||||
].join(' AND ');
|
||||
let sql = `
|
||||
SELECT song_id FROM wait_fetch_song WHERE ${whereClause}
|
||||
${args.order ? `ORDER BY song_id ${args.order}` : ''}
|
||||
${args.limit ? `LIMIT ${args.limit}` : ''}
|
||||
`;
|
||||
// // 更新现有数据
|
||||
// sql = `SELECT song_id FROM song WHERE data_version = 1`;
|
||||
console.log(sql);
|
||||
let songIds = await dbUtils.query(sql, []);
|
||||
songIds = songIds.map(item => item.song_id);
|
||||
return songIds;
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
album: {
|
||||
insert: async (albumInfo) => {
|
||||
return await dbUtils.query('INSERT IGNORE INTO album SET ?', albumInfo);
|
||||
@@ -29,21 +48,61 @@ module.exports = {
|
||||
|
||||
update: async (albumId, albumInfo) => {
|
||||
return await dbUtils.query(`UPDATE album SET ? WHERE album_id = ${albumId}`, albumInfo);
|
||||
}
|
||||
},
|
||||
|
||||
getIdsToFetch: async (args, isUpdate) => {
|
||||
let sql = "";
|
||||
if (isUpdate) {
|
||||
sql = `SELECT album_id FROM album WHERE (full_description = '' or full_description is null) and description like '%专辑《%》，简介：%' and description not regexp '^.*?专辑《.*?》，简介：[:space:]*?。，更多.*$'`;
|
||||
} else {
|
||||
let whereClause = [
|
||||
args.min ? `album_id > ${args.min}` : '1=1',
|
||||
args.max ? `album_id <= ${args.max}` : '1=1',
|
||||
].join(' AND ');
|
||||
sql = `
|
||||
SELECT album_id FROM wait_fetch_album WHERE ${whereClause}
|
||||
${args.order ? `ORDER BY album_id ${args.order}` : ''}
|
||||
${args.limit ? `LIMIT ${args.limit}` : ''}
|
||||
`;
|
||||
}
|
||||
console.log(sql);
|
||||
let albumIds = await dbUtils.query(sql, []);
|
||||
albumIds = albumIds.map(item => item.album_id);
|
||||
return albumIds;
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
artist: {
|
||||
insert: async (artistInfo) => {
|
||||
return await dbUtils.query('INSERT IGNORE INTO artist SET ?', artistInfo);
|
||||
},
|
||||
|
||||
getIdsToFetch: async (args) => {
|
||||
let whereClause = [
|
||||
args.min ? `artist_id > ${args.min}` : '1=1',
|
||||
args.max ? `artist_id <= ${args.max}` : '1=1',
|
||||
].join(' AND ');
|
||||
let sql = `
|
||||
SELECT artist_id FROM wait_fetch_artist WHERE ${whereClause}
|
||||
${args.order ? `ORDER BY artist_id ${args.order}` : ''}
|
||||
${args.limit ? `LIMIT ${args.limit}` : ''}
|
||||
`;
|
||||
console.log(sql);
|
||||
let artistIds = await dbUtils.query(sql, []);
|
||||
artistIds = artistIds.map(item => item.artist_id);
|
||||
return artistIds;
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
lyric: {
|
||||
insert: async (lyricInfo) => {
|
||||
return await dbUtils.query('INSERT IGNORE INTO lyric SET ?', lyricInfo);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
comment: {
|
||||
insertCollection: async (commentInfoList) => {
|
||||
if (commentInfoList.length == 0) return;
|
||||
@@ -51,15 +110,17 @@ module.exports = {
|
||||
INSERT INTO comment ( comment_id, parent_comment_id, user_id, song_id, content, time, like_count, comment_type ) VALUES ?
|
||||
ON DUPLICATE KEY UPDATE content = VALUES(content), like_count = VALUES(like_count), comment_type = GREATEST(comment_type, VALUES(comment_type)), modify_time = CURRENT_TIMESTAMP
|
||||
`, [commentInfoList]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
comment_progress: {
|
||||
update: async (commentProgressInfo, songId) => {
|
||||
return await dbUtils.query('UPDATE comment_progress SET ? WHERE song_id = ? LIMIT 1', [commentProgressInfo, songId]);
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
playlist: {
|
||||
insertCollection: async (playlistInfo) => {
|
||||
if (playlistInfo.length == 0) return;
|
||||
@@ -67,9 +128,10 @@ module.exports = {
|
||||
INSERT INTO playlist ( ${Object.keys(playlistInfo).map(field => `\`${field}\``).join(",")} ) VALUES ?
|
||||
ON DUPLICATE KEY UPDATE ${Object.keys(playlistInfo).map(field => `${field}=VALUES(${field})`).join(", ")}
|
||||
`, [[Object.values(playlistInfo)]]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
user: {
|
||||
insertCollection: async (userInfoList) => {
|
||||
if (userInfoList.length == 0) return;
|
||||
@@ -77,27 +139,44 @@ module.exports = {
|
||||
INSERT INTO user ( user_id, user_type, nickname, avatar_url ) VALUES ?
|
||||
ON DUPLICATE KEY UPDATE user_type = VALUES(user_type), nickname = VALUES(nickname), avatar_url = VALUES(avatar_url), modify_time = CURRENT_TIMESTAMP
|
||||
`, [userInfoList]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
song_album: {
|
||||
insertCollection: async (songAlbumRel) => {
|
||||
if (songAlbumRel.length == 0) return;
|
||||
return await dbUtils.query('INSERT IGNORE INTO song_album_relation (song_id, album_id) VALUES ?', [songAlbumRel]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
song_artist: {
|
||||
insertCollection: async (songArtistRel) => {
|
||||
if (songArtistRel.length == 0) return;
|
||||
return await dbUtils.query('INSERT IGNORE INTO song_artist_relation (song_id, artist_id) VALUES ?', [songArtistRel]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
song_playlist: {
|
||||
insertCollection: async (trackIds) => {
|
||||
if (trackIds.length == 0) return;
|
||||
return await dbUtils.query('INSERT IGNORE INTO song_playlist_relation (song_id, playlist_id, alg, rcmd_reason) VALUES ?', [trackIds]);
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
/* ##################################################### */
|
||||
|
||||
|
||||
// 将 id 插入待检查表
|
||||
wait_check: {
|
||||
insert: async (type, ids) => {
|
||||
// 过滤掉 id 为 0 的
|
||||
ids = ids.filter(id => id < 0);
|
||||
return await dbUtils.query(`INSERT IGNORE INTO wait_check_${type} (id) VALUES ?`, [ids]);
|
||||
},
|
||||
},
|
||||
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user