album，artist请求增加超时时间；小调整

hifini_music/index.js

@@ -75,6 +75,7 @@ async function startFetchDetail() {
         const threadId = idsToFetch[i];
         console.log(`getDetail\t| ${i + 1}/${idsToFetch.length} | threadId: ${threadId}`);
         await getDetail(threadId);
+        // await sleepUtils.sleep(100);
     }
 }
 
@@ -158,7 +159,7 @@ async function startFetchRealUrl() {
         const urlToFetch = urlsToFetch[i];
         console.log(`getRealUrl\t| ${i + 1}/${urlsToFetch.length} | threadId: ${urlToFetch.threadId}`);
         await getRealUrl(urlToFetch);
-        await sleepUtils.sleep(1000);
+        // await sleepUtils.sleep(100);
     }
 }
 

netease_music/sql/generator.js

@@ -0,0 +1,44 @@
+var table = [
+    "song",
+    "album",
+    "artist",
+    "comment",
+    "lyric",
+    "user",
+    "category",
+    "playlist",
+
+    "comment_progress",
+    "song_album_relation",
+    "song_artist_relation",
+    "song_playlist_relation",
+
+    "wait_check_album",
+    "wait_check_artist",
+    "wait_check_comment",
+    "wait_check_lyric",
+    "wait_check_song",
+
+    "wait_fetch_album",
+    "wait_fetch_artist",
+    "wait_fetch_lyric",
+    "wait_fetch_song",
+
+    "analysis",
+    "log",
+
+    "hifini_forum",
+    "hifini_tag",
+    "hifini_thread",
+    "hifini_thread_tag_relation",
+];
+
+let sqlList = [];
+
+// OPTIMIZE TABLE
+table.forEach((tableName) => sqlList.push(`OPTIMIZE TABLE ${tableName};`));
+
+// RENAME TABLE 移动数据库
+// table.forEach((tableName) => sqlList.push(`RENAME TABLE neteasemusic.${tableName} TO neteasemusic_develop.${tableName};`));
+
+console.log(sqlList.join('\n'));

netease_music/sql/statistic.sql

@@ -19,6 +19,27 @@ INSERT INTO analysis (`key`, `value`) VALUES ('songArtistCount', (SELECT count(*
 
 
 -- 更新后初次全表扫描
+INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_artist_relation   WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_album_relation    WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_playlist_relation WHERE create_time > '2022-10-28 00:00:00';
+
+INSERT IGNORE INTO wait_check_lyric   (id) SELECT song_id   FROM song_artist_relation   WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_lyric   (id) SELECT song_id   FROM song_album_relation    WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_lyric   (id) SELECT song_id   FROM song_playlist_relation WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_lyric   (id) SELECT song_id   FROM song                   WHERE create_time > '2022-10-28 00:00:00';
+
+INSERT IGNORE INTO wait_check_comment (id) SELECT song_id   FROM song_artist_relation   WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_comment (id) SELECT song_id   FROM song_album_relation    WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_comment (id) SELECT song_id   FROM song_playlist_relation WHERE create_time > '2022-10-28 00:00:00';
+INSERT IGNORE INTO wait_check_comment (id) SELECT song_id   FROM song                   WHERE create_time > '2022-10-28 00:00:00';
+
+INSERT IGNORE INTO wait_check_artist  (id) SELECT artist_id FROM song_artist_relation   WHERE create_time > '2022-10-28 00:00:00';
+
+INSERT IGNORE INTO wait_check_album   (id) SELECT album_id  FROM song_album_relation    WHERE create_time > '2022-10-28 00:00:00';
+
+
+
+-- 全量更新
 INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_artist_relation   WHERE song_id   NOT IN ( SELECT song_id   FROM song             );
 INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_album_relation    WHERE song_id   NOT IN ( SELECT song_id   FROM song             );
 INSERT IGNORE INTO wait_check_song    (id) SELECT song_id   FROM song_playlist_relation WHERE song_id   NOT IN ( SELECT song_id   FROM song             );
@@ -103,59 +124,6 @@ ORDER BY s DESC
 
 
 
--- optimize table
-optimize table analysis;
-optimize table album;
-optimize table artist;
-optimize table category;
-optimize table comment;
-optimize table comment_progress;
-optimize table log;
-optimize table lyric;
-optimize table playlist;
-optimize table song;
-optimize table song_album_relation;
-optimize table song_artist_relation;
-optimize table song_playlist_relation;
-optimize table user;
-optimize table wait_check_album;
-optimize table wait_check_artist;
-optimize table wait_check_lyric;
-optimize table wait_check_song;
-optimize table wait_fetch_album;
-optimize table wait_fetch_artist;
-optimize table wait_fetch_lyric;
-optimize table wait_fetch_song;
-
-
-
--- 移动数据库
-RENAME TABLE neteasemusic.analysis TO neteasemusic_develop.analysis;
-RENAME TABLE neteasemusic.album TO neteasemusic_develop.album;
-RENAME TABLE neteasemusic.artist TO neteasemusic_develop.artist;
-RENAME TABLE neteasemusic.category TO neteasemusic_develop.category;
-RENAME TABLE neteasemusic.comment TO neteasemusic_develop.comment;
-RENAME TABLE neteasemusic.comment_progress TO neteasemusic_develop.comment_progress;
-RENAME TABLE neteasemusic.log TO neteasemusic_develop.log;
-RENAME TABLE neteasemusic.lyric TO neteasemusic_develop.lyric;
-RENAME TABLE neteasemusic.playlist TO neteasemusic_develop.playlist;
-RENAME TABLE neteasemusic.song TO neteasemusic_develop.song;
-RENAME TABLE neteasemusic.song_album_relation TO neteasemusic_develop.song_album_relation;
-RENAME TABLE neteasemusic.song_artist_relation TO neteasemusic_develop.song_artist_relation;
-RENAME TABLE neteasemusic.song_playlist_relation TO neteasemusic_develop.song_playlist_relation;
-RENAME TABLE neteasemusic.user TO neteasemusic_develop.user;
-RENAME TABLE neteasemusic.wait_check_album TO neteasemusic_develop.wait_check_album;
-RENAME TABLE neteasemusic.wait_check_artist TO neteasemusic_develop.wait_check_artist;
-RENAME TABLE neteasemusic.wait_check_lyric TO neteasemusic_develop.wait_check_lyric;
-RENAME TABLE neteasemusic.wait_check_song TO neteasemusic_develop.wait_check_song;
-RENAME TABLE neteasemusic.wait_fetch_album TO neteasemusic_develop.wait_fetch_album;
-RENAME TABLE neteasemusic.wait_fetch_artist TO neteasemusic_develop.wait_fetch_artist;
-RENAME TABLE neteasemusic.wait_fetch_lyric TO neteasemusic_develop.wait_fetch_lyric;
-RENAME TABLE neteasemusic.wait_fetch_song TO neteasemusic_develop.wait_fetch_song;
-
-
-
-
 -- 查询单个数据库里面各个表所占磁盘空间大小包括其索引的大小
 SELECT
     table_schema AS '数据库',
@@ -170,28 +138,3 @@ WHERE
     table_schema = 'neteasemusic'
 ORDER BY
     table_rows DESC;
-
-
-
--- analysis
--- album
--- artist
--- category
--- comment
--- comment_progress
--- log
--- lyric
--- playlist
--- song
--- song_album_relation
--- song_artist_relation
--- song_playlist_relation
--- user
--- wait_check_album
--- wait_check_artist
--- wait_check_lyric
--- wait_check_song
--- wait_fetch_album
--- wait_fetch_artist
--- wait_fetch_lyric
--- wait_fetch_song

netease_music/src/assistantUtils.js

@@ -34,13 +34,14 @@ async function migrateIdsFromCheckToFetch(tableName, fieldName, insertSql = null
 
         // 插入待爬取列表
         if (finalIds.length > 0) {
-            await dbUtils.query(insertSql ? insertSql : `INSERT IGNORE INTO wait_fetch_${tableName} (id) VALUES ?`, [finalIds.map(id => [id])]);
+            var result = await dbUtils.query(insertSql ? insertSql : `INSERT IGNORE INTO wait_fetch_${tableName} (id) VALUES ?`, [finalIds.map(id => [id])]);
+            // console.log(result);
         }
 
         // 从待检查表中删除
         if (ids.length > 0)
             await dbUtils.query(`DELETE FROM wait_check_${tableName} WHERE id IN ?`, [[ids]]);
-        console.log(`table: ${tableName} | ${ids[0]} - ${ids.slice(-1)[0]}`);
+        console.log(`table: ${tableName} | ${ids[0]} - ${ids.slice(-1)[0]} (${result?.affectedRows}/${finalIds.length}/${ids.length})`);
     }
 }
 

netease_music/src/getInfo/albumInfoUtils.js

@@ -59,7 +59,7 @@ async function fetch({ albumId, debug = false, update = false }) {
     let url = `https://music.163.com/album?id=${albumId}`;
     try {
         // var html = fs.readFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), 'utf8');
-        var html = await requestUtils.getApiResult(url);
+        var html = await requestUtils.getApiResult(url, { timeout: 3000 });
         // fs.writeFileSync(path.join(__dirname, "../../temp", `album-${albumId}.html`), html);
     } catch (errors) {
         console.error(errors);

netease_music/src/getInfo/artistInfoUtils.js

@@ -52,7 +52,7 @@ async function fetch({ artistId, debug = false }) {
     let url = `https://music.163.com/artist?id=${artistId}`;
     try {
         // var html = fs.readFileSync(path.join(__dirname, "../../temp", `artist-${artistId}.html`), 'utf8');
-        var html = await requestUtils.getApiResult(url);
+        var html = await requestUtils.getApiResult(url, { timeout: 3000 });
         // fs.writeFileSync(path.join(__dirname, "../../temp", `artist-${artistId}.html`), html);
     } catch (errors) {
         console.error(errors);

netease_music/src/getInfo/songInfoUtils.js

@@ -21,7 +21,7 @@ async function fetchAll({ args = {} }) {
     for (let i = 0; i < count; i++) {
         await global.checkIsExit();
         var subArray = songIds.slice(i * step, (i + 1) * step);
-        console.log(`${i + 1}/${count} | song: ${subArray[0]}-${subArray.slice(-1)[0]} | ${args.min || "?"}-${args.max || "?"}`);
+        console.log(`${i + 1}/${count} | song: ${subArray[0]}-${subArray.slice(-1)[0]} (${subArray.length}) | ${args.min || "?"}-${args.max || "?"}`);
         try {
             await fetch({ songIdArray: subArray });
         } catch (err) {

netease_music/src/index.js

@@ -59,31 +59,37 @@ async function main(args) {
         switch (args.utils) {
             case 'song':
                 await songInfoUtils.fetchAll({ args: args });
+                await sleepUtils.sleep(60 * 1000);
                 break;
             case 'album':
                 await albumInfoUtils.fetchAll({ args: args });
+                await sleepUtils.sleep(30 * 1000);
                 break;
             case 'artist':
                 await artistInfoUtils.fetchAll({ args: args });
+                await sleepUtils.sleep(30 * 1000);
                 break;
             case 'lyric':
                 await lyricInfoUtils.fetchAll({ args: args });
+                await sleepUtils.sleep(30 * 1000);
                 break;
             case 'comment':
                 await commentUtils.fetchAll({ args: args });
+                await sleepUtils.sleep(30 * 1000);
                 break;
             case 'playlist':
                 await playlistUtils.fetchAll({ args: args });
+                process.exit(0);
                 break;
 
             case 'assistant':
                 await assistantUtils.updateWaitTable();
+                await sleepUtils.sleep(5000);
                 break;
             default:
                 console.log("utils参数不匹配，退出");
                 return;
         }
-        await sleepUtils.sleep(2000);
     }
 }