coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

更新SQL导出支持

Browse Source
This commit is contained in:
程序员小墨 2022-10-26 16:19:46 +08:00
parent aa5e239155
commit 66d5f89b02
10 changed files with 599 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
netease_music/sql/export/distribution_range/album.txt Normal file
View File

@ -0,0 +1,74 @@
154000000
153000000
152000000
151000000
150000000
149000000
148000000
147000000
146000000
145000000
144000000
143000000
142000000
141000000
140000000
139000000
138000000
137000000
136000000
135000000
134000000
133000000
132000000
131000000
130000000
129000000
128000000
127000000
126000000
125000000
124000000
123000000
122000000
121000000
120000000
99000000
98000000
97000000
96000000
95000000
94000000
93000000
92000000
91000000
90000000
89000000
88000000
87000000
86000000
85000000
84000000
83000000
82000000
81000000
80000000
79000000
78000000
77000000
76000000
75000000
74000000
73000000
72000000
40000000
39000000
38000000
37000000
36000000
35000000
34000000
3000000
2000000
1000000
0

16
netease_music/sql/export/distribution_range/artist.txt Normal file
View File

@ -0,0 +1,16 @@
54000000
52000000
50000000
48000000
46000000
38000000
36000000
34000000
32000000
30000000
28000000
16000000
14000000
12000000
2000000
0

70
netease_music/sql/export/distribution_range/song.txt Normal file
View File

@ -0,0 +1,70 @@
2000000000
1990000000
1980000000
1970000000
1960000000
1950000000
1940000000
1930000000
1920000000
1910000000
1900000000
1890000000
1880000000
1870000000
1860000000
1850000000
1840000000
1830000000
1820000000
1810000000
1800000000
1500000000
1490000000
1480000000
1470000000
1460000000
1450000000
1440000000
1430000000
1420000000
1410000000
1400000000
1390000000
1380000000
1370000000
1360000000
1350000000
1340000000
1330000000
1320000000
1310000000
1300000000
1290000000
870000000
860000000
580000000
570000000
560000000
550000000
540000000
530000000
520000000
510000000
500000000
490000000
480000000
470000000
460000000
450000000
440000000
430000000
420000000
410000000
400000000
40000000
30000000
20000000
10000000
5000000
0

317
netease_music/sql/export/distribution_range/user.txt Normal file
View File

@ -0,0 +1,317 @@
8080000000
8070000000
8060000000
8050000000
8040000000
8030000000
8020000000
8010000000
8000000000
7990000000
7980000000
7970000000
7960000000
7950000000
7940000000
7930000000
7920000000
7910000000
7900000000
7890000000
7880000000
7870000000
7860000000
7850000000
7840000000
7830000000
7820000000
7810000000
7800000000
7790000000
6490000000
6480000000
6470000000
6460000000
6450000000
6440000000
6430000000
6420000000
6410000000
6400000000
6390000000
6380000000
6370000000
6360000000
6350000000
6340000000
6330000000
6320000000
6310000000
6300000000
6290000000
6280000000
6270000000
5890000000
5220000000
5210000000
5200000000
5190000000
5180000000
5170000000
5160000000
5150000000
5140000000
5130000000
5120000000
5110000000
5100000000
5090000000
5080000000
5070000000
5060000000
5050000000
5040000000
5030000000
5020000000
5010000000
5000000000
4990000000
4980000000
4970000000
4960000000
4950000000
4940000000
4930000000
4920000000
4910000000
4900000000
4890000000
4880000000
4870000000
4060000000
4050000000
4040000000
4030000000
4020000000
4010000000
4000000000
3990000000
3980000000
3970000000
3960000000
3950000000
3940000000
3930000000
3920000000
3910000000
3900000000
3890000000
3880000000
3870000000
3860000000
3850000000
3840000000
3830000000
3820000000
3810000000
3800000000
3790000000
3780000000
3770000000
3760000000
3750000000
3740000000
3730000000
3720000000
3710000000
3700000000
3690000000
3680000000
3670000000
3660000000
3650000000
3640000000
3630000000
3620000000
3610000000
3600000000
3590000000
3580000000
3570000000
3560000000
3550000000
3540000000
3530000000
3520000000
3510000000
3500000000
3490000000
3480000000
3470000000
3460000000
3450000000
3440000000
3430000000
3420000000
3410000000
3400000000
3390000000
3380000000
3370000000
3360000000
3350000000
3340000000
3330000000
3320000000
3310000000
3300000000
3290000000
3280000000
3270000000
3260000000
3250000000
3240000000
3230000000
3220000000
2140000000
2130000000
2120000000
2110000000
2100000000
2090000000
2080000000
2070000000
2060000000
2050000000
2040000000
2030000000
2020000000
2010000000
2000000000
1990000000
1980000000
1970000000
1960000000
1950000000
1940000000
1930000000
1920000000
1910000000
1900000000
1890000000
1880000000
1870000000
1860000000
1850000000
1840000000
1830000000
1820000000
1810000000
1800000000
1790000000
1780000000
1770000000
1760000000
1750000000
1740000000
1730000000
1720000000
1710000000
1700000000
1690000000
1680000000
1670000000
1660000000
1650000000
1640000000
1630000000
1620000000
1610000000
1600000000
1590000000
1580000000
1570000000
1560000000
1550000000
1540000000
1530000000
1520000000
1510000000
1500000000
1490000000
1480000000
1470000000
1460000000
1450000000
1440000000
1430000000
1420000000
1410000000
1400000000
1390000000
1380000000
1370000000
1360000000
1350000000
1340000000
1330000000
1320000000
1310000000
1300000000
1290000000
650000000
640000000
630000000
620000000
610000000
600000000
590000000
580000000
570000000
560000000
550000000
540000000
530000000
520000000
510000000
500000000
490000000
480000000
470000000
460000000
450000000
440000000
430000000
420000000
410000000
400000000
390000000
380000000
370000000
360000000
350000000
340000000
330000000
320000000
310000000
300000000
290000000
280000000
270000000
260000000
250000000
140000000
130000000
120000000
110000000
100000000
90000000
80000000
70000000
60000000
50000000
40000000
30000000
20000000
10000000
0

83
netease_music/sql/export/export_sql_generator.js Normal file
View File

@ -0,0 +1,83 @@
const fs = require('fs');
const path = require('path');
const absPath = `D:/sql_export`;
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
const export_gap = 50000000;
const partition_gap = 50000000;
const table = "comment_1024";
const index = "comment_id";
const exportTablePrefix = "comment_export_";
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
let sqlArr1 = [];
let sqlArr2 = [];
let sqlArr3 = [];
for (let i = 200; i < 300; i++) {
// for (let i = 100; i < 200; i++) {
let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
sqlArr1.push(sql_create_table);
let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
sqlArr2.push(sql_delete_rows);
let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
sqlArr3.push(sql_partition);
}
sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// 建表
// console.log(sqlArr1.join('\n'));
// 删除原表数据
// console.log(sqlArr2.join('\n'));
// 新创建表的分区
// console.log(`partition (\n${sqlArr3.join(',\n')}\n)`);
// #############################################
// 删除表
// for (let i = 107; i < 200; i++) {
// console.log(`DROP TABLE IF EXISTS comment_export_${fill(i, '0', 4)};`);
// }
// #############################################
// 使用 mysqldump 分块导出数据表
let rangeTxtName = "album"; // 分布区间 "song" "album" "artist"
const dumpTable = "album"; // "comment_progress";
const fieldName = `album_id`;
var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim());
// console.log(a);
let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`];
for (let i = 0; i < a.length; i++) {
let where;
if (a[i + 1]) {
where = `${fieldName}>=${a[i]} and ${fieldName}<${a[i + 1]}`;
} else {
where = `${fieldName}>=${a[i]}`;
}
outputArr.push(`mysqldump neteasemusic -hrm-bp18qrc78dj7vd3newo.rwlb.rds.aliyuncs.com -uroot -pOj13EzoppxXvMmjPKh --tables ${dumpTable} --where="${where}" --skip-add-drop-table --set-gtid-purged=OFF > ${absPath}/${dumpTable}_${fill(i, '0', 4)}.sql`);
}
outputArr.push("echo done.");
console.log(outputArr.join('\n\n'));

44
netease_music/sql/export_sql_generator.js
View File

@ -1,44 +0,0 @@
const export_gap = 50000000;
const partition_gap = 50000000;
const table = "comment_1024";
const index = "comment_id";
const exportTablePrefix = "comment_export_";
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
let sqlArr1 = [];
let sqlArr2 = [];
let sqlArr3 = [];
for (let i = 200; i < 300; i++) {
// for (let i = 100; i < 200; i++) {
let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
sqlArr1.push(sql_create_table);
let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
sqlArr2.push(sql_delete_rows);
let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
sqlArr3.push(sql_partition);
}
sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// 建表
// console.log(sqlArr1.join('\n'));
// 删除原表数据
// console.log(sqlArr2.join('\n'));
// 新创建表的分区
// console.log(`partition (\n${sqlArr3.join(',\n')}\n)`);
for (let i = 107; i < 200; i++) {
console.log(`DROP TABLE IF EXISTS comment_export_${fill(i, '0', 4)};`);
}

6
netease_music/sql/neteasemusic.sql
View File

@ -194,17 +194,17 @@ CREATE TABLE `song` (
`fee` tinyint(4) NULL DEFAULT NULL COMMENT '版权 0: 免费或无版权 1: VIP 歌曲 4: 购买专辑 8: 非会员可免费播放低音质，会员可播放高音质及下载 fee 为 1 或 8 的歌曲均可单独购买 2 元单曲',
`quality` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '高/中/低/无损质量文件信息',
`cd` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'None或如\"04\", \"1/2\", \"3\", \"null\"的字符串，表示歌曲属于专辑中第几张CD，对应音频文件的Tag',
`no` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
`no` int(11) NULL DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
`dj_id` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '0: 不是DJ节目 其他：是DJ节目，表示DJ ID',
`s_id` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '对于t == 2的歌曲，表示匹配到的公开版本歌曲ID',
`origin_cover_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '0: 未知 1: 原曲 2: 翻唱',
`image` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '发布日期(弃用)',
`pub_time` bigint(20) UNSIGNED NULL DEFAULT NULL COMMENT '发布日期 毫秒为单位的Unix时间戳',
`pub_time` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '发布日期 毫秒为单位的Unix时间戳',
`no_copyright_rcmd` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'None表示可以播，非空表示无版权',
`mv` int(10) UNSIGNED NULL DEFAULT NULL COMMENT '非零表示有MV ID',
`single` tinyint(4) NULL DEFAULT NULL COMMENT '0: 有专辑信息或者是DJ节目 1: 未知专辑',
`version` int(10) UNSIGNED NOT NULL DEFAULT 1 COMMENT '歌曲版本信息',
`version` int(11) NOT NULL DEFAULT 1 COMMENT '歌曲版本信息',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
`data_version` tinyint(4) NOT NULL DEFAULT 1 COMMENT '数据记录版本(如果有字段调整则整体+1)',

33
netease_music/sql/statistic.sql
View File

@ -69,6 +69,39 @@ ORDER BY s DESC
-- 查看本地已有 song 的分布
SELECT cast( format( song_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count
FROM song
GROUP BY s
ORDER BY s DESC
-- 查看本地已有 user 的分布
SELECT cast( format( user_id / 10000000, 0) * 10000000 as UNSIGNED ) as s, count(*) as count
FROM user
GROUP BY s
ORDER BY s DESC
-- 查看本地已有 album 的分布
SELECT cast( format( album_id / 1000000, 0) * 1000000 as UNSIGNED ) as s, count(*) as count
FROM album
GROUP BY s
ORDER BY s DESC
-- 查看本地已有 artist 的分布
SELECT cast( format( artist_id / 2000000, 0) * 2000000 as UNSIGNED ) as s, count(*) as count
FROM artist
GROUP BY s
ORDER BY s DESC
-- 查看本地已有 playlist 的分布
SELECT cast( format( playlist_id / 2000000, 0) * 2000000 as UNSIGNED ) as s, count(*) as count
FROM playlist
GROUP BY s
ORDER BY s DESC
-- optimize table
optimize table analysis;
optimize table album;

6
netease_music/sql/structure.sql
View File

@ -9,17 +9,17 @@ CREATE TABLE `song` (
`fee` tinyint(4) DEFAULT NULL COMMENT '版权 0: 免费或无版权 1: VIP 歌曲 4: 购买专辑 8: 非会员可免费播放低音质，会员可播放高音质及下载 fee 为 1 或 8 的歌曲均可单独购买 2 元单曲',
`quality` varchar(500) NOT NULL COMMENT '高/中/低/无损质量文件信息',
`cd` varchar(255) NOT NULL COMMENT 'None或如"04", "1/2", "3", "null"的字符串，表示歌曲属于专辑中第几张CD，对应音频文件的Tag',
`no` int(10) unsigned DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
`no` int(11) DEFAULT NULL COMMENT '表示歌曲属于CD中第几曲，0表示没有这个字段，对应音频文件的Tag',
`dj_id` int(10) unsigned DEFAULT NULL COMMENT '0: 不是DJ节目 其他：是DJ节目，表示DJ ID',
`s_id` int(10) unsigned DEFAULT NULL COMMENT '对于t == 2的歌曲，表示匹配到的公开版本歌曲ID',
`origin_cover_type` tinyint(4) NOT NULL DEFAULT '0' COMMENT '0: 未知 1: 原曲 2: 翻唱',
`image` varchar(200) DEFAULT NULL COMMENT '封面图 http://p1.music.126.net/ 后面的部分',
`pub_date` varchar(100) DEFAULT NULL COMMENT '发布日期(弃用)',
`pub_time` bigint(20) unsigned DEFAULT NULL COMMENT '发布日期 毫秒为单位的Unix时间戳',
`pub_time` varchar(100) DEFAULT NULL COMMENT '发布日期 毫秒为单位的Unix时间戳',
`no_copyright_rcmd` varchar(255) DEFAULT NULL COMMENT 'None表示可以播，非空表示无版权',
`mv` int(10) unsigned DEFAULT NULL COMMENT '非零表示有MV ID',
`single` tinyint(4) DEFAULT NULL COMMENT '0: 有专辑信息或者是DJ节目 1: 未知专辑',
`version` int(10) unsigned NOT NULL DEFAULT '1' COMMENT '歌曲版本信息',
`version` int(11) NOT NULL DEFAULT '1' COMMENT '歌曲版本信息',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '爬取时间',
`modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间',
`data_version` tinyint(4) NOT NULL DEFAULT '1' COMMENT '数据记录版本(如果有字段调整则整体+1)',

0
netease_music/sql/replace_insert_table_name.js → netease_music/src/one_time_code/replace_insert_table_name.js
View File