coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

update

Browse Source
This commit is contained in:
程序员小墨 2022-10-01 18:53:19 +08:00
parent 64834ae7f9
commit 074cbc124f
7 changed files with 168 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
netease_music/index.js
View File

@ -15,6 +15,7 @@ async function main() {
// 指定数据库
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
// getMusicInfo({ songId: "1855221507" });
// getArtistInfo({ artistId: "1079074" });
@ -29,13 +30,12 @@ async function main() {
var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
console.log(affectRows1.affectedRows, affectRows2.affectedRows);
await startGetMusic(1);
await startGet(1);
await sleepUtils.sleep(2000);
}
}
async function startGetMusic(sleepTime) {
async function startGet(sleepTime) {
// 从数据库中查出还缺少的歌曲，并进行爬取
console.log("start fetching songs ...");
@ -80,7 +80,7 @@ async function startGetMusic(sleepTime) {
}
// 从数据库中查出还缺少的歌手，并进行爬取
console.log("start fetching albums ...")
console.log("start fetching artists ...")
let artistIds = await dbUtils.query(`
SELECT DISTINCT artist_id FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist )
`, []);
@ -100,29 +100,65 @@ async function startGetMusic(sleepTime) {
}
}
async function update() {
console.log("neteaseMusic update ...");
dbUtils.create("neteaseMusic");
console.log(`${await statistics()}`);
let sleepTime = 100;
// 从数据库中查出现有专辑，并进行更新
console.log("start fetching albums ...")
let albumIds = await dbUtils.query(`
SELECT DISTINCT album_id FROM album WHERE version = 1 -- and description like '%专辑《%》，简介：%'
`, []);
albumIds = albumIds.map(item => item.album_id);
for (let i = 0; i < albumIds.length; i++) {
const albumId = albumIds[i];
console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
try {
await albumInfoUtils.update({ albumId: albumId });
} catch (err) {
console.error(err);
}
await sleepUtils.sleep(sleepTime);
if (fs.readFileSync('stop.txt') == "1") {
throw new Error(`Stopped`);
}
}
}
async function statistics() {
let sql = `
SELECT
song_count,
album_count,
album_v3_count,
album_v4_count,
artist_count,
song_album_count,
song_artist_count
FROM
( SELECT count(*) AS song_count FROM song ) t1,
( SELECT count(*) AS album_count FROM album ) t2,
( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
( SELECT count(*) AS artist_count FROM artist ) t3,
( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
let result = await dbUtils.query(sql, []);
let songCount = result[0].song_count;
let albumCount = result[0].album_count;
let albumV3Count = result[0].album_v3_count;
let albumV4Count = result[0].album_v4_count;
let artistCount = result[0].artist_count;
let songAlbumCount = result[0].song_album_count;
let songArtistCount = result[0].song_artist_count;
return `song: ${songCount}, album: ${albumCount}, artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
}
module.exports = {
main: main,
update: update,
test: test,
}

35
netease_music/sql/structure.sql
View File

@ -2,9 +2,9 @@ CREATE DATABASE `neteaseMusic` CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_
USE `neteaseMusic`;
CREATE TABLE `song` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
`title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌曲名',
`image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
`title` varchar(200) NOT NULL COMMENT '歌曲名',
`image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) NOT NULL COMMENT '发布日期',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`song_id`)
@ -12,10 +12,10 @@ CREATE TABLE `song` (
CREATE TABLE `artist` (
`artist_id` int(10) unsigned NOT NULL COMMENT '歌手id',
`title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌手名',
`description` varchar(1500) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌手简介',
`image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
`title` varchar(200) NOT NULL COMMENT '歌手名',
`description` varchar(1500) NOT NULL COMMENT '歌手简介',
`image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) NOT NULL COMMENT '发布日期',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`artist_id`)
@ -23,13 +23,15 @@ CREATE TABLE `artist` (
CREATE TABLE `album` (
`album_id` int(10) unsigned NOT NULL COMMENT '专辑id',
`title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '专辑名',
`description` varchar(1500) COLLATE utf8mb4_general_ci NOT NULL COMMENT '专辑简介',
`image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
`company` varchar(100) COLLATE utf8mb4_general_ci NULL COMMENT '发行公司',
`title` varchar(200) NOT NULL COMMENT '专辑名',
`description` varchar(1500) NOT NULL COMMENT '专辑简介',
`full_description` text COMMENT '专辑简介(全)',
`image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) NOT NULL COMMENT '发布日期',
`company` varchar(100) DEFAULT NULL COMMENT '发行公司',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
`version` tinyint(4) NOT NULL DEFAULT '1' COMMENT '数据记录版本(如果有字段调整则整体+1)',
PRIMARY KEY (`album_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
@ -51,9 +53,16 @@ CREATE TABLE `song_artist_relation` (
CREATE TABLE `lyric` (
`song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
`lyric` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌词',
`lyric` text NOT NULL COMMENT '歌词',
`version` int(10) unsigned NOT NULL COMMENT '版本号',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
PRIMARY KEY (`song_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
CREATE TABLE `log` (
`id` int(10) unsigned NOT NULL COMMENT 'id',
`name` varchar(200) NOT NULL COMMENT '方法/数据库',
`msg` varchar(200) NOT NULL COMMENT '出错信息',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

103
netease_music/src/getInfo/albumInfoUtils.js
View File

@ -11,14 +11,6 @@ async function fetch({ albumId }) {
if (result[0].count > 0) {
console.log(`数据库中已有数据，跳过 albumId: ${albumId}`);
return;
// let albumResult = await dbUtils.query('SELECT * FROM album WHERE album_id = ?', [albumId]);
// albumResult = JSON.parse(JSON.stringify(albumResult));
// let songAlbumResult = await dbUtils.query('SELECT * FROM song_album_relation WHERE album_id = ?', [albumId]);
// songAlbumResult = JSON.parse(JSON.stringify(songAlbumResult));
// albumResult.songIds = songAlbumResult.map(song => song.song_id);
// // console.log(albumResult);
// return albumResult;
}
let url = `https://music.163.com/album?id=${albumId}`;
@ -31,7 +23,6 @@ async function fetch({ albumId }) {
console.error(errors);
return;
}
// console.log(html);
if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
// TODO 最后统一来处理这里 demo: artistId == 30084536
@ -44,10 +35,40 @@ async function fetch({ albumId }) {
let albumInfoDict = JSON.parse(albumInfoJSONString);
// console.log(albumInfoDict);
// 发行公司
let company = null;
if (html.includes(`<p class="intr"><b>发行公司：`)) {
try {
company = /<p class="intr"><b>发行公司：<\/b>\n(.*?)\n<\/p>/.exec(html)[1];
} catch (e) {
// 解析出错
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `company 正则失败\n${e.message}`]);
return;
}
}
// 专辑详细简介
let fullDescription = null;
if (html.includes(`<div id="album-desc-more" class="f-hide">`)) {
// 比较长 有点击展开按钮
try {
fullDescription = /<div id="album-desc-more" class="f-hide">([\S\s]*?)<\/div>/.exec(html)[1];
fullDescription = fullDescription.replace(/<p class="f-brk">\n/g, '').replace(/<\/p>\n/g, '').trim();
} catch (e) {
// 解析出错
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 1 正则失败\n${e.message}`]);
return;
}
} else if (html.includes(`<div id="album-desc-dot" class="f-brk">`)) {
// 比较短 无点击展开按钮
try {
fullDescription = /<div id="album-desc-dot" class="f-brk">([\S\s]*?)<\/div>/.exec(html)[1];
fullDescription = fullDescription.replace(/<p>/g, '').replace(/<\/p>/g, '').trim();
} catch (e) {
// 解析出错
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 2 正则失败\n${e.message}`]);
return;
}
}
let image = /<meta property="og:image" content="http:\/\/p.\.music\.126\.net\/(.*?)" \/>/.exec(html)[1];
@ -60,6 +81,7 @@ async function fetch({ albumId }) {
title: albumInfoDict.title,
image: image,
description: albumInfoDict.description,
full_description: fullDescription,
pubDate: albumInfoDict.pubDate,
company: company,
songIds: songIds,
@ -69,9 +91,11 @@ async function fetch({ albumId }) {
album_id: albumInfo.albumId,
title: albumInfo.title,
description: albumInfo.description,
full_description: albumInfo.fullDescription,
image: albumInfo.image,
pub_date: albumInfo.pubDate,
company: albumInfo.company,
version: 4
});
songIds.forEach(function (songId) {
if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0)
@ -84,6 +108,67 @@ async function fetch({ albumId }) {
return albumInfo;
}
/*
v1 to v3
升级v3完毕后应该查不出记录才对
SELECT
*
FROM
album
WHERE
version = 3 and full_description is null and description like '%专辑《%》，简介：%'
*/
async function update({ albumId }) {
let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
if (result[0].count == 0) {
console.log(`数据库中没有数据，跳过 albumId: ${albumId}`);
return;
}
let url = `https://music.163.com/album?id=${albumId}`;
try {
// var html = fs.readFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), 'utf8');
var html = await requestUtils.getApiResult(url);
// fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
} catch (errors) {
console.error(errors);
return;
}
if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
return;
}
// 专辑详细简介
let fullDescription = null;
if (html.includes(`<div id="album-desc-more" class="f-hide">`)) {
try {
fullDescription = /<div id="album-desc-more" class="f-hide">([\S\s]*?)<\/div>/.exec(html)[1];
fullDescription = fullDescription.replace(/<p class="f-brk">\n/g, '').replace(/<\/p>\n/g, '').trim();
} catch (e) {
// 解析出错
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 3 正则失败\n${e.message}`]);
return;
}
} else if (html.includes(`<div id="album-desc-dot" class="f-brk">`)) {
try {
fullDescription = /<div id="album-desc-dot" class="f-brk">([\S\s]*?)<\/div>/.exec(html)[1];
fullDescription = fullDescription.replace(/<p>/g, '').replace(/<\/p>/g, '').trim();
} catch (e) {
// 解析出错
await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 4 正则失败\n${e.message}`]);
return;
}
}
await dbUtils.query('UPDATE album SET full_description = ?, version = 3 WHERE album_id = ?', [fullDescription, albumId]);
return;
}
module.exports = {
fetch: fetch,
update: update,
}

1
netease_music/src/getInfo/artistInfoUtils.js
View File

@ -32,7 +32,6 @@ async function fetch({ artistId }) {
console.error(errors);
return;
}
// console.log(html);
if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
// TODO 最后统一来处理这里 demo: artistId == 30084536

1
netease_music/src/getInfo/songInfoUtils.js
View File

@ -37,7 +37,6 @@ async function fetch({ songId }) {
console.error(errors);
return;
}
// console.log(html);
if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
// TODO 最后统一来处理这里 demo: artistId == 30084536

1
netease_music/src/getInfo/userInfoUtils.js
View File

@ -15,7 +15,6 @@ async function fetch({ userId }) {
var html = await requestUtils.getApiResult(url);
fs.writeFileSync(path.join(__dirname, "../../temp", ` user-${userId}.html`), html);
}
// console.log(html);
}

2
update.js Normal file
View File

@ -0,0 +1,2 @@
const neteaseMusic = require('./netease_music/index');
neteaseMusic.update();