update node watch

netease_music/sql/statistic.sql

@@ -1,3 +1,20 @@
+INSERT INTO analysis (`key`, `value`) VALUES ('songCount', (SELECT count(*) as count FROM song) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('songWaiting', (SELECT count(DISTINCT song_id) AS count
+                FROM ( SELECT song_id FROM song_artist_relation UNION SELECT song_id FROM song_album_relation ) t_tmp
+                WHERE song_id NOT IN ( SELECT song_id FROM song )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('albumCount', (SELECT count(*) as count FROM album) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('albumWaiting', (SELECT count( DISTINCT album_id ) as count FROM song_album_relation WHERE album_id NOT IN ( SELECT album_id FROM album )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('artistCount', (SELECT count(*) AS count FROM artist) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('artistWaiting', (SELECT count( DISTINCT artist_id ) as count FROM song_artist_relation WHERE artist_id NOT IN ( SELECT artist_id FROM artist )) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('lyricCount', (SELECT count(*) AS count FROM lyric) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('commentCount', (SELECT count( DISTINCT song_id ) AS count FROM comment) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('commentTotalCount', (SELECT count(*) AS count FROM comment) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('userCount', (SELECT count(*) AS count FROM user) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('songAlbumCount', (SELECT count(*) AS count FROM song_album_relation) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+INSERT INTO analysis (`key`, `value`) VALUES ('songArtistCount', (SELECT count(*) AS count FROM song_artist_relation) ) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
+
+
+
 -- 查看需要爬取的 song 的分布
 SELECT cast( format( t_tmp.song_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count
 FROM (
@@ -38,6 +55,7 @@ GROUP BY s
 ORDER BY s DESC
 
 
+
 -- optimize table
 optimize table album;
 optimize table artist;
@@ -50,6 +68,8 @@ optimize table song_album_relation;
 optimize table song_artist_relation;
 optimize table user;
 
+
+
 -- 查询单个数据库里面各个表所占磁盘空间大小包括其索引的大小
 SELECT
     table_schema AS '数据库',

netease_music/sql/structure.sql

@@ -130,3 +130,10 @@ CREATE TABLE `log` (
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   KEY `id` (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE `analysis` (
+  `key` varchar(255) NOT NULL COMMENT '参数名',
+  `value` varchar(255) DEFAULT NULL COMMENT '参数值',
+  `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
+  UNIQUE KEY `key` (`key`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

netease_music/src/index.js

@@ -143,7 +143,8 @@ async function watch() {
         promiseList.push(new Promise(async (resolve, reject) => {
             // console.log(`query ${sql.name} ...`);
             let sqlStartTime = Date.now();
-            let result = await dbUtils.query(sql.sql, []);
+            // let result = await dbUtils.query(sql.sql, []);
+            let result = await dbUtils.query(`SELECT \`value\` as count FROM analysis WHERE \`key\`='${sql.name}'`);
             let sqlTimeSpent = Date.now() - sqlStartTime;
             sqlsTimeSpent += sqlTimeSpent;
             newWatchParam[sql.name] = result[0].count;

netease_music/watch.js

@@ -1,4 +1,4 @@
-let keepWatching = true;
+let keepWatching = false;
 if (keepWatching) {
     global.useMysqlPool = true;
     global.connectionLimit = 15;