const fs = require('fs'); const path = require('path'); const requestUtils = require('../../../utils/requestUtils'); const sleepUtils = require('../../../utils/sleepUtils'); const dbUtils = global.dbUtils; // 从数据库中查询 async function getFromDatabase({ songId }) { // 查询出专辑 let infoResultSet = await dbUtils.query('SELECT * FROM song WHERE song_id = ?', [songId]); if (infoResultSet.length == 0) return {}; // 查出专辑与歌曲对应关系 let albumRelationResultSet = await dbUtils.query('SELECT * FROM song_album_relation WHERE song_id = ?', [songId]); let artistRelationResultSet = await dbUtils.query('SELECT * FROM song_artist_relation WHERE song_id = ?', [songId]); // 拼装 let songInfo = JSON.parse(JSON.stringify(infoResultSet[0])); songInfo.albumIds = albumRelationResultSet.map(album => album.album_id); songInfo.artistIds = artistRelationResultSet.map(artist => artist.artist_id); return songInfo; } // 从数据库中查出还缺少的歌曲,并进行爬取 async function fetchAll() { console.log("start fetching songs ..."); var songIds = await dbUtils.query(` SELECT DISTINCT song_id FROM song_artist_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) UNION SELECT DISTINCT song_id FROM song_album_relation WHERE song_id NOT IN ( SELECT DISTINCT song_id FROM song ) `, []); songIds = songIds.map(item => item.song_id); for (let i = 0; i < songIds.length; i++) { await global.checkIsExit(); const songId = songIds[i]; console.log(`${i + 1}/${songIds.length} | song: ${songId}`); try { await fetch({ songId: songId }); } catch (err) { console.error(err); } await sleepUtils.sleep(global.sleepTime); } } // 获取音乐详情 async function fetch({ songId, debug = false }) { let result = await dbUtils.query('SELECT count(*) as count FROM song WHERE song_id = ?', [songId]); if (result[0].count > 0 && !debug) { console.log(`数据库中已有数据,跳过 songId: ${songId}`); return; } let url = `https://music.163.com/song?id=${songId}`; try { // var html = fs.readFileSync(path.join(__dirname, "../../temp", `song-${songId}.html`), 'utf8'); var html = await requestUtils.getApiResult(url); // fs.writeFileSync(path.join(__dirname, "../../temp", `song-${songId}.html`), html); } catch (errors) { console.error(errors); return; } if (html.includes(`
很抱歉,你要查找的网页找不到
`)) { let deleteResult1 = await dbUtils.query('DELETE FROM song_album_relation WHERE song_id = ?', [songId]); let deleteResult2 = await dbUtils.query('DELETE FROM song_artist_relation WHERE song_id = ?', [songId]); console.log(`song: ${songId} 不存在,从song_album_relation, song_artist_relation表中删除. affectedRows: ${deleteResult1.affectedRows}, ${deleteResult2.affectedRows}`); return; } // 正则匹配 let regExResult = /\