update

netease_music/index.js

@@ -15,6 +15,7 @@ async function main() {
 
     // 指定数据库
     dbUtils.create("neteaseMusic");
+    console.log(`${await statistics()}`);
 
     // getMusicInfo({ songId: "1855221507" });
     // getArtistInfo({ artistId: "1079074" });
@@ -29,13 +30,12 @@ async function main() {
         var affectRows2 = await dbUtils.query(`DELETE FROM song_album_relation WHERE song_id = 0 OR album_id = 0`, []);
         console.log(affectRows1.affectedRows, affectRows2.affectedRows);
 
-        await startGetMusic(1);
+        await startGet(1);
-
         await sleepUtils.sleep(2000);
     }
 }
 
-async function startGetMusic(sleepTime) {
+async function startGet(sleepTime) {
 
     // 从数据库中查出还缺少的歌曲，并进行爬取
     console.log("start fetching songs ...");
@@ -80,7 +80,7 @@ async function startGetMusic(sleepTime) {
     }
 
     // 从数据库中查出还缺少的歌手，并进行爬取
-    console.log("start fetching albums ...")
+    console.log("start fetching artists ...")
     let artistIds = await dbUtils.query(`
         SELECT DISTINCT artist_id FROM song_artist_relation WHERE artist_id NOT IN ( SELECT DISTINCT artist_id FROM artist )
     `, []);
@@ -100,29 +100,65 @@ async function startGetMusic(sleepTime) {
     }
 }
 
+async function update() {
+    console.log("neteaseMusic update ...");
+    dbUtils.create("neteaseMusic");
+    console.log(`${await statistics()}`);
+
+    let sleepTime = 100;
+
+    // 从数据库中查出现有专辑，并进行更新
+    console.log("start fetching albums ...")
+    let albumIds = await dbUtils.query(`
+        SELECT DISTINCT album_id FROM album WHERE version = 1 -- and description like '%专辑《%》，简介：%'
+    `, []);
+    albumIds = albumIds.map(item => item.album_id);
+    for (let i = 0; i < albumIds.length; i++) {
+        const albumId = albumIds[i];
+        console.log(`${i}/${albumIds.length} | album: ${albumId} | ${await statistics()}`);
+        try {
+            await albumInfoUtils.update({ albumId: albumId });
+        } catch (err) {
+            console.error(err);
+        }
+        await sleepUtils.sleep(sleepTime);
+        if (fs.readFileSync('stop.txt') == "1") {
+            throw new Error(`Stopped`);
+        }
+    }
+}
+
 async function statistics() {
     let sql = `
         SELECT
             song_count,
             album_count,
+            album_v3_count,
+            album_v4_count,
             artist_count,
             song_album_count,
             song_artist_count
         FROM
             ( SELECT count(*) AS song_count FROM song ) t1,
             ( SELECT count(*) AS album_count FROM album ) t2,
+            ( SELECT count(*) AS album_v3_count FROM album WHERE version = 3 ) t3_2,
+            ( SELECT count(*) AS album_v4_count FROM album WHERE version = 4 ) t4_2,
             ( SELECT count(*) AS artist_count FROM artist ) t3,
             ( SELECT count(*) AS song_album_count FROM song_album_relation ) t4,
             ( SELECT count(*) AS song_artist_count FROM song_artist_relation ) t5`;
     let result = await dbUtils.query(sql, []);
     let songCount = result[0].song_count;
     let albumCount = result[0].album_count;
+    let albumV3Count = result[0].album_v3_count;
+    let albumV4Count = result[0].album_v4_count;
     let artistCount = result[0].artist_count;
     let songAlbumCount = result[0].song_album_count;
     let songArtistCount = result[0].song_artist_count;
-    return `song: ${songCount}, album: ${albumCount}, artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
+    return `song: ${songCount}, album: ${albumCount}(v3: ${albumV3Count}, v4: ${albumV4Count}), artist: ${artistCount} | songAlbum: ${songAlbumCount}, songArtist: ${songArtistCount}`;
 }
 
 module.exports = {
     main: main,
+    update: update,
+    test: test,
 }

netease_music/sql/structure.sql

@@ -2,9 +2,9 @@ CREATE DATABASE `neteaseMusic` CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_general_
 USE `neteaseMusic`;
 CREATE TABLE `song` (
   `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
-  `title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌曲名',
+  `title` varchar(200) NOT NULL COMMENT '歌曲名',
-  `image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
+  `image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
-  `pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
+  `pub_date` varchar(100) NOT NULL COMMENT '发布日期',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
   PRIMARY KEY (`song_id`)
@@ -12,10 +12,10 @@ CREATE TABLE `song` (
 
 CREATE TABLE `artist` (
   `artist_id` int(10) unsigned NOT NULL COMMENT '歌手id',
-  `title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌手名',
+  `title` varchar(200) NOT NULL COMMENT '歌手名',
-  `description` varchar(1500) COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌手简介',
+  `description` varchar(1500) NOT NULL COMMENT '歌手简介',
-  `image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
+  `image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
-  `pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
+  `pub_date` varchar(100) NOT NULL COMMENT '发布日期',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
   PRIMARY KEY (`artist_id`)
@@ -23,13 +23,15 @@ CREATE TABLE `artist` (
 
 CREATE TABLE `album` (
   `album_id` int(10) unsigned NOT NULL COMMENT '专辑id',
-  `title` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '专辑名',
+  `title` varchar(200) NOT NULL COMMENT '专辑名',
-  `description` varchar(1500) COLLATE utf8mb4_general_ci NOT NULL COMMENT '专辑简介',
+  `description` varchar(1500) NOT NULL COMMENT '专辑简介',
-  `image` varchar(200) COLLATE utf8mb4_general_ci NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
+  `full_description` text COMMENT '专辑简介(全)',
-  `pub_date` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '发布日期',
+  `image` varchar(200) NOT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
-  `company` varchar(100) COLLATE utf8mb4_general_ci NULL COMMENT '发行公司',
+  `pub_date` varchar(100) NOT NULL COMMENT '发布日期',
+  `company` varchar(100) DEFAULT NULL COMMENT '发行公司',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
+  `version` tinyint(4) NOT NULL DEFAULT '1' COMMENT '数据记录版本(如果有字段调整则整体+1)',
   PRIMARY KEY (`album_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
 
@@ -51,9 +53,16 @@ CREATE TABLE `song_artist_relation` (
 
 CREATE TABLE `lyric` (
   `song_id` int(10) unsigned NOT NULL COMMENT '歌曲id',
-  `lyric` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '歌词',
+  `lyric` text NOT NULL COMMENT '歌词',
   `version` int(10) unsigned NOT NULL COMMENT '版本号',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
   PRIMARY KEY (`song_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE `log` (
+  `id` int(10) unsigned NOT NULL COMMENT 'id',
+  `name` varchar(200) NOT NULL COMMENT '方法/数据库',
+  `msg` varchar(200) NOT NULL COMMENT '出错信息',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间'
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

netease_music/src/getInfo/albumInfoUtils.js

@@ -11,14 +11,6 @@ async function fetch({ albumId }) {
     if (result[0].count > 0) {
         console.log(`数据库中已有数据，跳过 albumId: ${albumId}`);
         return;
-
-        // let albumResult = await dbUtils.query('SELECT * FROM album WHERE album_id = ?', [albumId]);
-        // albumResult = JSON.parse(JSON.stringify(albumResult));
-        // let songAlbumResult = await dbUtils.query('SELECT * FROM song_album_relation WHERE album_id = ?', [albumId]);
-        // songAlbumResult = JSON.parse(JSON.stringify(songAlbumResult));
-        // albumResult.songIds = songAlbumResult.map(song => song.song_id);
-        // // console.log(albumResult);
-        // return albumResult;
     }
 
     let url = `https://music.163.com/album?id=${albumId}`;
@@ -31,7 +23,6 @@ async function fetch({ albumId }) {
         console.error(errors);
         return;
     }
-    // console.log(html);
 
     if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
         // TODO 最后统一来处理这里 demo: artistId == 30084536
@@ -44,10 +35,40 @@ async function fetch({ albumId }) {
     let albumInfoDict = JSON.parse(albumInfoJSONString);
     // console.log(albumInfoDict);
 
+    // 发行公司
     let company = null;
+    if (html.includes(`<p class="intr"><b>发行公司：`)) {
         try {
             company = /<p class="intr"><b>发行公司：<\/b>\n(.*?)\n<\/p>/.exec(html)[1];
         } catch (e) {
+            // 解析出错
+            await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `company 正则失败\n${e.message}`]);
+            return;
+        }
+    }
+
+    // 专辑详细简介
+    let fullDescription = null;
+    if (html.includes(`<div id="album-desc-more" class="f-hide">`)) {
+        // 比较长 有点击展开按钮
+        try {
+            fullDescription = /<div id="album-desc-more" class="f-hide">([\S\s]*?)<\/div>/.exec(html)[1];
+            fullDescription = fullDescription.replace(/<p class="f-brk">\n/g, '').replace(/<\/p>\n/g, '').trim();
+        } catch (e) {
+            // 解析出错
+            await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 1 正则失败\n${e.message}`]);
+            return;
+        }
+    } else if (html.includes(`<div id="album-desc-dot" class="f-brk">`)) {
+        // 比较短 无点击展开按钮
+        try {
+            fullDescription = /<div id="album-desc-dot" class="f-brk">([\S\s]*?)<\/div>/.exec(html)[1];
+            fullDescription = fullDescription.replace(/<p>/g, '').replace(/<\/p>/g, '').trim();
+        } catch (e) {
+            // 解析出错
+            await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 2 正则失败\n${e.message}`]);
+            return;
+        }
     }
 
     let image = /<meta property="og:image" content="http:\/\/p.\.music\.126\.net\/(.*?)" \/>/.exec(html)[1];
@@ -60,6 +81,7 @@ async function fetch({ albumId }) {
         title: albumInfoDict.title,
         image: image,
         description: albumInfoDict.description,
+        full_description: fullDescription,
         pubDate: albumInfoDict.pubDate,
         company: company,
         songIds: songIds,
@@ -69,9 +91,11 @@ async function fetch({ albumId }) {
         album_id: albumInfo.albumId,
         title: albumInfo.title,
         description: albumInfo.description,
+        full_description: albumInfo.fullDescription,
         image: albumInfo.image,
         pub_date: albumInfo.pubDate,
         company: albumInfo.company,
+        version: 4
     });
     songIds.forEach(function (songId) {
         if (isNaN(Number(songId)) || Number(songId) === 0 || isNaN(Number(albumId)) || Number(songId) === 0)
@@ -84,6 +108,67 @@ async function fetch({ albumId }) {
     return albumInfo;
 }
 
+/*
+v1 to v3
+
+升级v3完毕后应该查不出记录才对
+SELECT
+    *
+FROM
+    album
+WHERE
+    version = 3 and full_description is null and description like '%专辑《%》，简介：%'
+*/
+async function update({ albumId }) {
+    let result = await dbUtils.query('SELECT count(*) as count FROM album WHERE album_id = ?', [albumId]);
+    if (result[0].count == 0) {
+        console.log(`数据库中没有数据，跳过 albumId: ${albumId}`);
+        return;
+    }
+
+    let url = `https://music.163.com/album?id=${albumId}`;
+
+    try {
+        // var html = fs.readFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), 'utf8');
+        var html = await requestUtils.getApiResult(url);
+        // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
+    } catch (errors) {
+        console.error(errors);
+        return;
+    }
+
+    if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
+        return;
+    }
+
+    // 专辑详细简介
+    let fullDescription = null;
+    if (html.includes(`<div id="album-desc-more" class="f-hide">`)) {
+        try {
+            fullDescription = /<div id="album-desc-more" class="f-hide">([\S\s]*?)<\/div>/.exec(html)[1];
+            fullDescription = fullDescription.replace(/<p class="f-brk">\n/g, '').replace(/<\/p>\n/g, '').trim();
+        } catch (e) {
+            // 解析出错
+            await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 3 正则失败\n${e.message}`]);
+            return;
+        }
+    } else if (html.includes(`<div id="album-desc-dot" class="f-brk">`)) {
+        try {
+            fullDescription = /<div id="album-desc-dot" class="f-brk">([\S\s]*?)<\/div>/.exec(html)[1];
+            fullDescription = fullDescription.replace(/<p>/g, '').replace(/<\/p>/g, '').trim();
+        } catch (e) {
+            // 解析出错
+            await dbUtils.query('INSERT INTO log (`id`, `name`, `msg`) VALUES (?, ?, ?)', [albumId, 'album_fetch', `fullDescription 4 正则失败\n${e.message}`]);
+            return;
+        }
+    }
+
+    await dbUtils.query('UPDATE album SET full_description = ?, version = 3 WHERE album_id = ?', [fullDescription, albumId]);
+    return;
+}
+
+
 module.exports = {
     fetch: fetch,
+    update: update,
 }

netease_music/src/getInfo/artistInfoUtils.js

@@ -32,7 +32,6 @@ async function fetch({ artistId }) {
         console.error(errors);
         return;
     }
-    // console.log(html);
 
     if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
         // TODO 最后统一来处理这里 demo: artistId == 30084536

netease_music/src/getInfo/songInfoUtils.js

@@ -37,7 +37,6 @@ async function fetch({ songId }) {
         console.error(errors);
         return;
     }
-    // console.log(html);
 
     if (html.includes(`<p class="note s-fc3">很抱歉，你要查找的网页找不到</p>`)) {
         // TODO 最后统一来处理这里 demo: artistId == 30084536

netease_music/src/getInfo/userInfoUtils.js

@@ -15,7 +15,6 @@ async function fetch({ userId }) {
         var html = await requestUtils.getApiResult(url);
         fs.writeFileSync(path.join(__dirname, "../../temp", ` user-${userId}.html`), html);
     }
-    // console.log(html);
 
 
 }

update.js

@@ -0,0 +1,2 @@
+const neteaseMusic = require('./netease_music/index');
+neteaseMusic.update();