123 lines
6.5 KiB
JavaScript
123 lines
6.5 KiB
JavaScript
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const sleepUtils = require('../../../utils/sleepUtils');
|
||
|
||
const dbUtils = global.dbUtils;
|
||
|
||
const { song_detail } = require('NeteaseCloudMusicApi');
|
||
|
||
// 从数据库中查出还缺少的歌曲，并进行爬取
|
||
async function fetchAll({ args = {} }) {
|
||
console.log("start fetching songs ...");
|
||
let whereClause = [
|
||
args.min ? `song_id > ${args.min}` : '1=1',
|
||
args.max ? `song_id <= ${args.max}` : '1=1',
|
||
].join(' AND ');
|
||
var sql = `
|
||
SELECT DISTINCT song_id FROM song_artist_relation WHERE ${whereClause} AND song_id NOT IN ( SELECT song_id FROM song )
|
||
UNION
|
||
SELECT DISTINCT song_id FROM song_album_relation WHERE ${whereClause} AND song_id NOT IN ( SELECT song_id FROM song )
|
||
${args.order ? `ORDER BY song_id ${args.order}` : ''}
|
||
${args.limit ? `LIMIT ${args.limit}` : ''}
|
||
`;
|
||
// // 更新现有数据
|
||
// sql = `SELECT song_id FROM song WHERE ${whereClause} AND data_version = 1`;
|
||
// 测试用
|
||
// sql = `SELECT song_id FROM song_artist_relation group by song_id limit 10`;
|
||
console.log(sql);
|
||
|
||
var songIds = await dbUtils.query(sql, []);
|
||
songIds = songIds.map(item => item.song_id);
|
||
|
||
// 0 - 100, 200 - 399, 400 - ..., ... - songIds.length-1
|
||
// 0 1 2 count-1
|
||
var step = 1000;
|
||
var total = songIds.length;
|
||
var count = Math.ceil(total / step);
|
||
for (let i = 0; i < count; i++) {
|
||
await global.checkIsExit();
|
||
var subArray = songIds.slice(i * step, (i + 1) * step);
|
||
console.log(`${i + 1}/${count} | song: ${subArray[0]}-${subArray.slice(-1)[0]} | ${args.min || "?"}-${args.max || "?"}`);
|
||
try {
|
||
await fetch({ songIdArray: subArray });
|
||
} catch (err) {
|
||
console.error(err);
|
||
}
|
||
await sleepUtils.sleep(global.sleepTime);
|
||
}
|
||
}
|
||
|
||
// 获取音乐详情
|
||
async function fetch({ songIdArray, debug = false }) {
|
||
// https://neteasecloudmusicapi-docs.4everland.app/#/?id=%e8%8e%b7%e5%8f%96%e6%ad%8c%e6%9b%b2%e8%af%a6%e6%83%85
|
||
try {
|
||
// 每一次大概可以取到1000条以上
|
||
var songResult = await song_detail({ ids: songIdArray.join(',') });
|
||
// fs.writeFileSync(path.join(__dirname, "../../temp", `song-${songIdArray[0]}-${songIdArray[songIdArray.length - 1]}.json`), JSON.stringify(songResult));
|
||
} catch (errors) {
|
||
console.error(errors);
|
||
return;
|
||
}
|
||
// console.log(songResult.body.songs.map(item => JSON.stringify(item)));
|
||
|
||
let songAlbumRel = [], songArtistRel = [];
|
||
let songInfoList = songResult.body.songs.map(song => {
|
||
song.ar.forEach(item => songArtistRel.push([song.id, item.id]));
|
||
songAlbumRel.push([song.id, song.al.id || 0])
|
||
return {
|
||
title: song.name, // 歌曲标题
|
||
id: song.id, // 歌曲ID
|
||
type: song.t, // 0: 一般类型 1: 通过云盘上传的音乐，网易云不存在公开对应 2: 通过云盘上传的音乐，网易云存在公开对应
|
||
alias: JSON.stringify(song.alia), // 别名列表，第一个别名会被显示作副标题
|
||
pop: song.pop, // 小数，常取[0.0, 100.0]中离散的几个数值, 表示歌曲热度
|
||
fee: song.fee, // 版权 0: 免费或无版权 1: VIP 歌曲 4: 购买专辑 8: 非会员可免费播放低音质，会员可播放高音质及下载 fee 为 1 或 8 的歌曲均可单独购买 2 元单曲
|
||
duration: song.dt, // 歌曲时长
|
||
quality: JSON.stringify({ h: song.h, m: song.m, l: song.l, sq: song.sq }), // 高/中/低/无损质量文件信息
|
||
version: song.version, // 歌曲版本信息
|
||
cd: song.cd, // None或如"04", "1/2", "3", "null"的字符串，表示歌曲属于专辑中第几张CD，对应音频文件的Tag
|
||
no: song.no, // 表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag
|
||
djId: song.djId, // 0: 不是DJ节目 其他：是DJ节目，表示DJ ID
|
||
sId: song.s_id, // 对于t == 2的歌曲，表示匹配到的公开版本歌曲ID
|
||
originCoverType: song.originCoverType, // 0: 未知 1: 原曲 2: 翻唱
|
||
image: "",
|
||
pubTime: song.publishTime, // 毫秒为单位的Unix时间戳
|
||
mv: song.mv, // 非零表示有MV ID
|
||
single: song.single, // 0: 有专辑信息或者是DJ节目 1: 未知专辑
|
||
noCopyrightRcmd: song.noCopyrightRcmd ? JSON.stringify(song.noCopyrightRcmd) : "", // None表示可以播，非空表示无版权
|
||
|
||
artist: song.ar.map(item => item.id), // 歌手列表
|
||
album: song.al.id || 0, // 专辑，如果是DJ节目(dj_type != 0)或者无专辑信息(single == 1)，则专辑id为0
|
||
};
|
||
});
|
||
|
||
// console.log("songAlbumRel, songArtistRel", songAlbumRel, songArtistRel);
|
||
// console.log("songInfoList", songInfoList);
|
||
|
||
if (songInfoList.length == 0) return;
|
||
|
||
console.log("插入数据库");
|
||
if (songAlbumRel.length > 0)
|
||
await dbUtils.query('INSERT IGNORE INTO song_album_relation (song_id, album_id) VALUES ?', [songAlbumRel]);
|
||
await dbUtils.query('INSERT IGNORE INTO song_artist_relation (song_id, artist_id) VALUES ?', [songArtistRel]);
|
||
await dbUtils.query(`
|
||
INSERT INTO song (
|
||
song_id, title, type, alias, pop, fee, quality, cd,
|
||
no, dj_id, s_id, origin_cover_type, pub_time,
|
||
no_copyright_rcmd, mv, single, version, data_version
|
||
) VALUES ? ON DUPLICATE KEY UPDATE
|
||
title = VALUES(title), type = VALUES(type), alias = VALUES(alias), pop = VALUES(pop), fee = VALUES(fee), quality = VALUES(quality), cd = VALUES(cd),
|
||
no = VALUES(no), dj_id = VALUES(dj_id), s_id = VALUES(s_id), origin_cover_type = VALUES(origin_cover_type), pub_time = VALUES(pub_time),
|
||
no_copyright_rcmd = VALUES(no_copyright_rcmd), mv = VALUES(mv), single = VALUES(single), version = VALUES(version), data_version = VALUES(data_version)
|
||
`, [songInfoList.map(songInfo => [
|
||
songInfo.id, songInfo.title, songInfo.type, songInfo.alias, songInfo.pop, songInfo.fee, songInfo.quality, songInfo.cd,
|
||
songInfo.no, songInfo.djId, songInfo.sId, songInfo.originCoverType, songInfo.pubTime,
|
||
songInfo.noCopyrightRcmd, songInfo.mv, songInfo.single, songInfo.version, 2
|
||
])]);
|
||
// image 因为接口没有返回，所以不更新
|
||
}
|
||
|
||
module.exports = {
|
||
fetch: fetch,
|
||
fetchAll: fetchAll,
|
||
} |