coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

爬取歌单 简单代码

Browse Source
This commit is contained in:
程序员小墨 2022-10-19 17:22:49 +08:00
parent 8451e7a849
commit d864d2ab5f
5 changed files with 437 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
qianqian_music/data/.gitkeep Normal file
View File

61
qianqian_music/index.js Normal file
View File

@ -0,0 +1,61 @@
const fs = require('fs');
const path = require('path');
const requestUtils = require('../utils/requestUtils');
const TEMP_MODULE_PATH = path.join(__dirname, 'temp');
const DATA_PATH = path.join(__dirname, 'data');
/**
* 获取 html
*/
async function getHtml(url) {
// let html = fs.readFileSync("songlist");
let html = await requestUtils.getApiResult(url, {
gzip: true,
});
return html;
}
/**
* 正则匹配出JSON
*/
function getData(html, index = 0) {
// 首先匹配出 window.__NUXT__
let script = /<script>window.__NUXT__=(.*?);<\/script>/.exec(html)[1];
// 然后将该匿名函数作为模块引入，取得解析后的JSON
let tempModule = `${TEMP_MODULE_PATH}/temp_module_${index}.js`;
fs.writeFileSync(tempModule, `exports.data=${script};`);
try {
let data = require(tempModule);
// console.log(data);
// data 下有分类 和 歌单信息 这里只取歌单信息部分
return data.data.data[0].listData;
} catch (err) {
console.error(err);
} finally {
fs.rmSync(tempModule);
}
}
async function main() {
let currentPage = 1;
let haveMore = true;
while (haveMore) {
let url = `https://music.91q.com/songlist?pageNo=${currentPage}`;
console.log(url);
let html = await getHtml(url);
fs.writeFileSync(`${DATA_PATH}/songlist-html-${currentPage}.html`, html);
console.log("get html done.");
let data = getData(html, currentPage);
fs.writeFileSync(`${DATA_PATH}/songlist-data-${currentPage}.json`, JSON.stringify(data, null, 2));
console.log("get data done.");
currentPage++;
haveMore = data.haveMore;
}
}
main();

371
qianqian_music/one_time_code/category_convert_json_to_sql.js Normal file
View File

@ -0,0 +1,371 @@
var json = {
"language": [
{
"categoryName": "华语",
"id": "2368",
"count": 94
},
{
"categoryName": "欧美",
"id": "2369",
"count": 84
},
{
"categoryName": "粤语",
"id": "2370",
"count": 2
},
{
"categoryName": "日语",
"id": "2371",
"count": 71
},
{
"categoryName": "韩语",
"id": "2372",
"count": 1
},
{
"categoryName": "小语种",
"id": "2374",
"count": 6
}
],
"genre": [
{
"categoryName": "流行",
"id": "2376",
"count": 247
},
{
"categoryName": "摇滚",
"id": "2377",
"count": 66
},
{
"categoryName": "民谣",
"id": "2378",
"count": 10
},
{
"categoryName": "电子",
"id": "2379",
"count": 63
},
{
"categoryName": "影视原声",
"id": "2380",
"count": 15
},
{
"categoryName": "ACG",
"id": "2381",
"count": 18
},
{
"categoryName": "新世纪",
"id": "2383",
"count": 5
},
{
"categoryName": "爵士",
"id": "2384",
"count": 44
},
{
"categoryName": "古典",
"id": "2385",
"count": 5
},
{
"categoryName": "乡村",
"id": "2386",
"count": 49
},
{
"categoryName": "说唱",
"id": "2387",
"count": 34
},
{
"categoryName": "世界音乐",
"id": "2388",
"count": 16
},
{
"categoryName": "古风",
"id": "2389",
"count": 10
},
{
"categoryName": "儿歌",
"id": "2390",
"count": 7
},
{
"categoryName": "朋克",
"id": "2391",
"count": 3
},
{
"categoryName": "布鲁斯",
"id": "2392",
"count": 23
},
{
"categoryName": "RnB/Soul",
"id": "2393",
"count": 12
},
{
"categoryName": "金属",
"id": "2394",
"count": 3
},
{
"categoryName": "英伦",
"id": "2396",
"count": 3
},
{
"categoryName": "民族",
"id": "2397",
"count": 18
},
{
"categoryName": "拉丁",
"id": "2399",
"count": 6
}
],
"mood": [
{
"categoryName": "快乐",
"id": "2401",
"count": 30
},
{
"categoryName": "美好",
"id": "2402",
"count": 62
},
{
"categoryName": "安静",
"id": "2403",
"count": 42
},
{
"categoryName": "寂寞",
"id": "2405",
"count": 1
},
{
"categoryName": "思念",
"id": "2406",
"count": 20
},
{
"categoryName": "孤独",
"id": "2407",
"count": 6
},
{
"categoryName": "怀旧",
"id": "2408",
"count": 14
},
{
"categoryName": "悲伤",
"id": "2409",
"count": 23
},
{
"categoryName": "感动",
"id": "2410",
"count": 10
},
{
"categoryName": "治愈",
"id": "2411",
"count": 28
},
{
"categoryName": "放松",
"id": "2412",
"count": 96
},
{
"categoryName": "清新",
"id": "2413",
"count": 22
},
{
"categoryName": "浪漫",
"id": "2414",
"count": 19
},
{
"categoryName": "兴奋",
"id": "2415",
"count": 34
},
{
"categoryName": "性感",
"id": "2416",
"count": 52
},
{
"categoryName": "励志",
"id": "2417",
"count": 30
},
{
"categoryName": "纯净",
"id": "2455",
"count": 2
}
],
"scene": [
{
"categoryName": "运动",
"id": "2419",
"count": 39
},
{
"categoryName": "驾驶",
"id": "2420",
"count": 21
},
{
"categoryName": "学习",
"id": "2421",
"count": 19
},
{
"categoryName": "工作",
"id": "2422",
"count": 6
},
{
"categoryName": "清晨",
"id": "2423",
"count": 17
},
{
"categoryName": "夜晚",
"id": "2424",
"count": 74
},
{
"categoryName": "午后",
"id": "2425",
"count": 31
},
{
"categoryName": "游戏",
"id": "2426",
"count": 3
},
{
"categoryName": "旅行",
"id": "2427",
"count": 51
},
{
"categoryName": "散步",
"id": "2428",
"count": 14
},
{
"categoryName": "酒吧",
"id": "2429",
"count": 9
},
{
"categoryName": "夜店",
"id": "2430",
"count": 9
},
{
"categoryName": "咖啡厅",
"id": "2431",
"count": 9
},
{
"categoryName": "地铁",
"id": "2432",
"count": 3
},
{
"categoryName": "校园",
"id": "2433",
"count": 6
},
{
"categoryName": "约会",
"id": "2435",
"count": 4
},
{
"categoryName": "休息",
"id": "2436",
"count": 15
},
{
"categoryName": "减压",
"id": "2456",
"count": 17
}
],
"theme": [
{
"categoryName": "经典",
"id": "2438",
"count": 9
},
{
"categoryName": "翻唱",
"id": "2439",
"count": 1
},
{
"categoryName": "KTV",
"id": "2442",
"count": 2
},
{
"categoryName": "网络歌曲",
"id": "2444",
"count": 2
},
{
"categoryName": "器乐",
"id": "2445",
"count": 90
},
{
"categoryName": "节日",
"id": "2446",
"count": 129
},
{
"categoryName": "人群",
"id": "2447",
"count": 11
},
{
"categoryName": "年代",
"id": "2448",
"count": 5
}
],
};
var a = [];
Object.keys(json).forEach(function (key) {
json[key].map(cate => {
a.push(`(${cate.id}, '${cate.categoryName}', '${key}')`);
});
});
console.log(`INSERT INTO category (qianqian_id, title, \`group\`) VALUES ${a.join(', ')}`);

5
qianqian_music/one_time_code/category_update_group_chinese.sql Normal file
View File

@ -0,0 +1,5 @@
UPDATE category SET `qianqian_group_chinese` = '语种' WHERE `qianqian_group` = 'language';
UPDATE category SET `qianqian_group_chinese` = '流派' WHERE `qianqian_group` = 'genre';
UPDATE category SET `qianqian_group_chinese` = '主题' WHERE `qianqian_group` = 'theme';
UPDATE category SET `qianqian_group_chinese` = '情感' WHERE `qianqian_group` = 'mood';
UPDATE category SET `qianqian_group_chinese` = '场景' WHERE `qianqian_group` = 'scene';

0
qianqian_music/temp/.gitkeep Normal file
View File