更新SQL导出支持

netease_music/sql/export/distribution_range/album.txt

@@ -0,0 +1,74 @@
+154000000
+153000000
+152000000
+151000000
+150000000
+149000000
+148000000
+147000000
+146000000
+145000000
+144000000
+143000000
+142000000
+141000000
+140000000
+139000000
+138000000
+137000000
+136000000
+135000000
+134000000
+133000000
+132000000
+131000000
+130000000
+129000000
+128000000
+127000000
+126000000
+125000000
+124000000
+123000000
+122000000
+121000000
+120000000
+99000000
+98000000
+97000000
+96000000
+95000000
+94000000
+93000000
+92000000
+91000000
+90000000
+89000000
+88000000
+87000000
+86000000
+85000000
+84000000
+83000000
+82000000
+81000000
+80000000
+79000000
+78000000
+77000000
+76000000
+75000000
+74000000
+73000000
+72000000
+40000000
+39000000
+38000000
+37000000
+36000000
+35000000
+34000000
+3000000
+2000000
+1000000
+0

netease_music/sql/export/distribution_range/artist.txt

@@ -0,0 +1,16 @@
+54000000
+52000000
+50000000
+48000000
+46000000
+38000000
+36000000
+34000000
+32000000
+30000000
+28000000
+16000000
+14000000
+12000000
+2000000
+0

netease_music/sql/export/distribution_range/song.txt

@@ -0,0 +1,70 @@
+2000000000
+1990000000
+1980000000
+1970000000
+1960000000
+1950000000
+1940000000
+1930000000
+1920000000
+1910000000
+1900000000
+1890000000
+1880000000
+1870000000
+1860000000
+1850000000
+1840000000
+1830000000
+1820000000
+1810000000
+1800000000
+1500000000
+1490000000
+1480000000
+1470000000
+1460000000
+1450000000
+1440000000
+1430000000
+1420000000
+1410000000
+1400000000
+1390000000
+1380000000
+1370000000
+1360000000
+1350000000
+1340000000
+1330000000
+1320000000
+1310000000
+1300000000
+1290000000
+870000000
+860000000
+580000000
+570000000
+560000000
+550000000
+540000000
+530000000
+520000000
+510000000
+500000000
+490000000
+480000000
+470000000
+460000000
+450000000
+440000000
+430000000
+420000000
+410000000
+400000000
+40000000
+30000000
+20000000
+10000000
+5000000
+0

netease_music/sql/export/distribution_range/user.txt

@@ -0,0 +1,317 @@
+8080000000
+8070000000
+8060000000
+8050000000
+8040000000
+8030000000
+8020000000
+8010000000
+8000000000
+7990000000
+7980000000
+7970000000
+7960000000
+7950000000
+7940000000
+7930000000
+7920000000
+7910000000
+7900000000
+7890000000
+7880000000
+7870000000
+7860000000
+7850000000
+7840000000
+7830000000
+7820000000
+7810000000
+7800000000
+7790000000
+6490000000
+6480000000
+6470000000
+6460000000
+6450000000
+6440000000
+6430000000
+6420000000
+6410000000
+6400000000
+6390000000
+6380000000
+6370000000
+6360000000
+6350000000
+6340000000
+6330000000
+6320000000
+6310000000
+6300000000
+6290000000
+6280000000
+6270000000
+5890000000
+5220000000
+5210000000
+5200000000
+5190000000
+5180000000
+5170000000
+5160000000
+5150000000
+5140000000
+5130000000
+5120000000
+5110000000
+5100000000
+5090000000
+5080000000
+5070000000
+5060000000
+5050000000
+5040000000
+5030000000
+5020000000
+5010000000
+5000000000
+4990000000
+4980000000
+4970000000
+4960000000
+4950000000
+4940000000
+4930000000
+4920000000
+4910000000
+4900000000
+4890000000
+4880000000
+4870000000
+4060000000
+4050000000
+4040000000
+4030000000
+4020000000
+4010000000
+4000000000
+3990000000
+3980000000
+3970000000
+3960000000
+3950000000
+3940000000
+3930000000
+3920000000
+3910000000
+3900000000
+3890000000
+3880000000
+3870000000
+3860000000
+3850000000
+3840000000
+3830000000
+3820000000
+3810000000
+3800000000
+3790000000
+3780000000
+3770000000
+3760000000
+3750000000
+3740000000
+3730000000
+3720000000
+3710000000
+3700000000
+3690000000
+3680000000
+3670000000
+3660000000
+3650000000
+3640000000
+3630000000
+3620000000
+3610000000
+3600000000
+3590000000
+3580000000
+3570000000
+3560000000
+3550000000
+3540000000
+3530000000
+3520000000
+3510000000
+3500000000
+3490000000
+3480000000
+3470000000
+3460000000
+3450000000
+3440000000
+3430000000
+3420000000
+3410000000
+3400000000
+3390000000
+3380000000
+3370000000
+3360000000
+3350000000
+3340000000
+3330000000
+3320000000
+3310000000
+3300000000
+3290000000
+3280000000
+3270000000
+3260000000
+3250000000
+3240000000
+3230000000
+3220000000
+2140000000
+2130000000
+2120000000
+2110000000
+2100000000
+2090000000
+2080000000
+2070000000
+2060000000
+2050000000
+2040000000
+2030000000
+2020000000
+2010000000
+2000000000
+1990000000
+1980000000
+1970000000
+1960000000
+1950000000
+1940000000
+1930000000
+1920000000
+1910000000
+1900000000
+1890000000
+1880000000
+1870000000
+1860000000
+1850000000
+1840000000
+1830000000
+1820000000
+1810000000
+1800000000
+1790000000
+1780000000
+1770000000
+1760000000
+1750000000
+1740000000
+1730000000
+1720000000
+1710000000
+1700000000
+1690000000
+1680000000
+1670000000
+1660000000
+1650000000
+1640000000
+1630000000
+1620000000
+1610000000
+1600000000
+1590000000
+1580000000
+1570000000
+1560000000
+1550000000
+1540000000
+1530000000
+1520000000
+1510000000
+1500000000
+1490000000
+1480000000
+1470000000
+1460000000
+1450000000
+1440000000
+1430000000
+1420000000
+1410000000
+1400000000
+1390000000
+1380000000
+1370000000
+1360000000
+1350000000
+1340000000
+1330000000
+1320000000
+1310000000
+1300000000
+1290000000
+650000000
+640000000
+630000000
+620000000
+610000000
+600000000
+590000000
+580000000
+570000000
+560000000
+550000000
+540000000
+530000000
+520000000
+510000000
+500000000
+490000000
+480000000
+470000000
+460000000
+450000000
+440000000
+430000000
+420000000
+410000000
+400000000
+390000000
+380000000
+370000000
+360000000
+350000000
+340000000
+330000000
+320000000
+310000000
+300000000
+290000000
+280000000
+270000000
+260000000
+250000000
+140000000
+130000000
+120000000
+110000000
+100000000
+90000000
+80000000
+70000000
+60000000
+50000000
+40000000
+30000000
+20000000
+10000000
+0

netease_music/sql/export/export_sql_generator.js

@@ -0,0 +1,83 @@
+const fs = require('fs');
+const path = require('path');
+
+
+const absPath = `D:/sql_export`;
+
+// 数字转成字符串，同时在前面填充
+function fill(num, fillers, length) {
+    var result = `${num}`;
+    if (result.length < length)
+        result = new Array(length - result.length + 1).join(fillers) + result;
+    return result;
+}
+
+
+const export_gap = 50000000;
+const partition_gap = 50000000;
+const table = "comment_1024";
+const index = "comment_id";
+const exportTablePrefix = "comment_export_";
+
+// 数字转成字符串，同时在前面填充
+function fill(num, fillers, length) {
+    var result = `${num}`;
+    if (result.length < length)
+        result = new Array(length - result.length + 1).join(fillers) + result;
+    return result;
+}
+
+let sqlArr1 = [];
+let sqlArr2 = [];
+let sqlArr3 = [];
+for (let i = 200; i < 300; i++) {
+    // for (let i = 100; i < 200; i++) {
+    let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
+
+    let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
+    sqlArr1.push(sql_create_table);
+
+    let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
+    sqlArr2.push(sql_delete_rows);
+
+    let sql_partition = `  PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
+    sqlArr3.push(sql_partition);
+}
+sqlArr3.push(`  PARTITION p_max VALUES LESS THAN MAXVALUE`);
+
+// 建表
+// console.log(sqlArr1.join('\n'));
+
+// 删除原表数据
+// console.log(sqlArr2.join('\n'));
+
+// 新创建表的分区
+// console.log(`partition (\n${sqlArr3.join(',\n')}\n)`);
+
+// #############################################
+
+// 删除表
+// for (let i = 107; i < 200; i++) {
+//     console.log(`DROP TABLE IF EXISTS comment_export_${fill(i, '0', 4)};`);
+// }
+
+// #############################################
+
+// 使用 mysqldump 分块导出数据表
+let rangeTxtName = "album"; // 分布区间 "song" "album" "artist"
+const dumpTable = "album"; // "comment_progress";
+const fieldName = `album_id`;
+var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim());
+// console.log(a);
+let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`];
+for (let i = 0; i < a.length; i++) {
+    let where;
+    if (a[i + 1]) {
+        where = `${fieldName}>=${a[i]} and ${fieldName}<${a[i + 1]}`;
+    } else {
+        where = `${fieldName}>=${a[i]}`;
+    }
+    outputArr.push(`mysqldump neteasemusic -hrm-bp18qrc78dj7vd3newo.rwlb.rds.aliyuncs.com -uroot -pOj13EzoppxXvMmjPKh --tables ${dumpTable} --where="${where}" --skip-add-drop-table --set-gtid-purged=OFF > ${absPath}/${dumpTable}_${fill(i, '0', 4)}.sql`);
+}
+outputArr.push("echo done.");
+console.log(outputArr.join('\n\n'));

netease_music/sql/export_sql_generator.js

@@ -1,44 +0,0 @@
-const export_gap = 50000000;
-const partition_gap = 50000000;
-const table = "comment_1024";
-const index = "comment_id";
-const exportTablePrefix = "comment_export_";
-
-// 数字转成字符串，同时在前面填充
-function fill(num, fillers, length) {
-    var result = `${num}`;
-    if (result.length < length)
-        result = new Array(length - result.length + 1).join(fillers) + result;
-    return result;
-}
-
-let sqlArr1 = [];
-let sqlArr2 = [];
-let sqlArr3 = [];
-for (let i = 200; i < 300; i++) {
-// for (let i = 100; i < 200; i++) {
-    let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
-
-    let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
-    sqlArr1.push(sql_create_table);
-
-    let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
-    sqlArr2.push(sql_delete_rows);
-
-    let sql_partition = `  PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
-    sqlArr3.push(sql_partition);
-}
-sqlArr3.push(`  PARTITION p_max VALUES LESS THAN MAXVALUE`);
-
-// 建表
-// console.log(sqlArr1.join('\n'));
-
-// 删除原表数据
-// console.log(sqlArr2.join('\n'));
-
-// 新创建表的分区
-// console.log(`partition (\n${sqlArr3.join(',\n')}\n)`);
-
-for (let i = 107; i < 200; i++) {
-    console.log(`DROP TABLE IF EXISTS comment_export_${fill(i, '0', 4)};`);
-}

netease_music/sql/neteasemusic.sql

@@ -194,17 +194,17 @@ CREATE TABLE `song`  (
   `fee` tinyint(4) NULL DEFAULT NULL COMMENT '版权  0: 免费或无版权  1: VIP 歌曲  4: 购买专辑  8: 非会员可免费播放低音质，会员可播放高音质及下载  fee 为 1 或 8 的歌曲均可单独购买 2 元单曲',
   `quality` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '高/中/低/无损质量文件信息',
   `cd` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'None或如\"04\", \"1/2\", \"3\", \"null\"的字符串，表示歌曲属于专辑中第几张CD，对应音频文件的Tag',
-  `no` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
+  `no` int(11) NULL DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
   `dj_id` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '0: 不是DJ节目 其他：是DJ节目，表示DJ ID',
   `s_id` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '对于t == 2的歌曲，表示匹配到的公开版本歌曲ID',
   `origin_cover_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '0: 未知  1: 原曲  2: 翻唱',
   `image` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
   `pub_date` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '发布日期(弃用)',
-  `pub_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '发布日期  毫秒为单位的Unix时间戳',
+  `pub_time` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '发布日期  毫秒为单位的Unix时间戳',
   `no_copyright_rcmd` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'None表示可以播，非空表示无版权',
   `mv` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '非零表示有MV ID',
   `single` tinyint(4) NULL DEFAULT NULL COMMENT '0: 有专辑信息或者是DJ节目  1: 未知专辑',
-  `version` int(10) UNSIGNED NOT NULL DEFAULT 1 COMMENT '歌曲版本信息',
+  `version` int(11) NOT NULL DEFAULT 1 COMMENT '歌曲版本信息',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
   `data_version` tinyint(4) NOT NULL DEFAULT 1 COMMENT '数据记录版本(如果有字段调整则整体+1)',

netease_music/sql/statistic.sql

@@ -69,6 +69,39 @@ ORDER BY s DESC
 
 
 
+
+-- 查看本地已有 song 的分布
+SELECT cast( format( song_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count
+FROM song
+GROUP BY s
+ORDER BY s DESC
+
+-- 查看本地已有 user 的分布
+SELECT cast( format( user_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count
+FROM user
+GROUP BY s
+ORDER BY s DESC
+
+-- 查看本地已有 album 的分布
+SELECT cast( format( album_id / 1000000, 0) * 1000000 as UNSIGNED ) as s, count(*) as count
+FROM album
+GROUP BY s
+ORDER BY s DESC
+
+-- 查看本地已有 artist 的分布
+SELECT cast( format( artist_id / 2000000, 0) * 2000000 as UNSIGNED ) as s, count(*) as count
+FROM artist
+GROUP BY s
+ORDER BY s DESC
+
+-- 查看本地已有 playlist 的分布
+SELECT cast( format( playlist_id / 2000000, 0) * 2000000 as UNSIGNED ) as s, count(*) as count
+FROM playlist
+GROUP BY s
+ORDER BY s DESC
+
+
+
 -- optimize table
 optimize table analysis;
 optimize table album;

netease_music/sql/structure.sql

@@ -9,17 +9,17 @@ CREATE TABLE `song` (
   `fee` tinyint(4) DEFAULT NULL COMMENT '版权  0: 免费或无版权  1: VIP 歌曲  4: 购买专辑  8: 非会员可免费播放低音质，会员可播放高音质及下载  fee 为 1 或 8 的歌曲均可单独购买 2 元单曲',
   `quality` varchar(500) NOT NULL COMMENT '高/中/低/无损质量文件信息',
   `cd` varchar(255) NOT NULL COMMENT 'None或如"04", "1/2", "3", "null"的字符串，表示歌曲属于专辑中第几张CD，对应音频文件的Tag',
-  `no` int(10) unsigned DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
+  `no` int(11) DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
   `dj_id` int(10) unsigned DEFAULT NULL COMMENT '0: 不是DJ节目 其他：是DJ节目，表示DJ ID',
   `s_id` int(10) unsigned DEFAULT NULL COMMENT '对于t == 2的歌曲，表示匹配到的公开版本歌曲ID',
   `origin_cover_type` tinyint(4) NOT NULL DEFAULT '0' COMMENT '0: 未知  1: 原曲  2: 翻唱',
   `image` varchar(200) DEFAULT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
   `pub_date` varchar(100) DEFAULT NULL COMMENT '发布日期(弃用)',
-  `pub_time` bigint(20) unsigned DEFAULT NULL COMMENT '发布日期  毫秒为单位的Unix时间戳',
+  `pub_time` varchar(100) DEFAULT NULL COMMENT '发布日期  毫秒为单位的Unix时间戳',
   `no_copyright_rcmd` varchar(255) DEFAULT NULL COMMENT 'None表示可以播，非空表示无版权',
   `mv` int(10) unsigned DEFAULT NULL COMMENT '非零表示有MV ID',
   `single` tinyint(4) DEFAULT NULL COMMENT '0: 有专辑信息或者是DJ节目  1: 未知专辑',
-  `version` int(10) unsigned NOT NULL DEFAULT '1' COMMENT '歌曲版本信息',
+  `version` int(11) NOT NULL DEFAULT '1' COMMENT '歌曲版本信息',
   `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
   `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
   `data_version` tinyint(4) NOT NULL DEFAULT '1' COMMENT '数据记录版本(如果有字段调整则整体+1)',