更新SQL导出支持
This commit is contained in:
74
netease_music/sql/export/distribution_range/album.txt
Normal file
74
netease_music/sql/export/distribution_range/album.txt
Normal file
@@ -0,0 +1,74 @@
|
||||
154000000
|
||||
153000000
|
||||
152000000
|
||||
151000000
|
||||
150000000
|
||||
149000000
|
||||
148000000
|
||||
147000000
|
||||
146000000
|
||||
145000000
|
||||
144000000
|
||||
143000000
|
||||
142000000
|
||||
141000000
|
||||
140000000
|
||||
139000000
|
||||
138000000
|
||||
137000000
|
||||
136000000
|
||||
135000000
|
||||
134000000
|
||||
133000000
|
||||
132000000
|
||||
131000000
|
||||
130000000
|
||||
129000000
|
||||
128000000
|
||||
127000000
|
||||
126000000
|
||||
125000000
|
||||
124000000
|
||||
123000000
|
||||
122000000
|
||||
121000000
|
||||
120000000
|
||||
99000000
|
||||
98000000
|
||||
97000000
|
||||
96000000
|
||||
95000000
|
||||
94000000
|
||||
93000000
|
||||
92000000
|
||||
91000000
|
||||
90000000
|
||||
89000000
|
||||
88000000
|
||||
87000000
|
||||
86000000
|
||||
85000000
|
||||
84000000
|
||||
83000000
|
||||
82000000
|
||||
81000000
|
||||
80000000
|
||||
79000000
|
||||
78000000
|
||||
77000000
|
||||
76000000
|
||||
75000000
|
||||
74000000
|
||||
73000000
|
||||
72000000
|
||||
40000000
|
||||
39000000
|
||||
38000000
|
||||
37000000
|
||||
36000000
|
||||
35000000
|
||||
34000000
|
||||
3000000
|
||||
2000000
|
||||
1000000
|
||||
0
|
||||
16
netease_music/sql/export/distribution_range/artist.txt
Normal file
16
netease_music/sql/export/distribution_range/artist.txt
Normal file
@@ -0,0 +1,16 @@
|
||||
54000000
|
||||
52000000
|
||||
50000000
|
||||
48000000
|
||||
46000000
|
||||
38000000
|
||||
36000000
|
||||
34000000
|
||||
32000000
|
||||
30000000
|
||||
28000000
|
||||
16000000
|
||||
14000000
|
||||
12000000
|
||||
2000000
|
||||
0
|
||||
70
netease_music/sql/export/distribution_range/song.txt
Normal file
70
netease_music/sql/export/distribution_range/song.txt
Normal file
@@ -0,0 +1,70 @@
|
||||
2000000000
|
||||
1990000000
|
||||
1980000000
|
||||
1970000000
|
||||
1960000000
|
||||
1950000000
|
||||
1940000000
|
||||
1930000000
|
||||
1920000000
|
||||
1910000000
|
||||
1900000000
|
||||
1890000000
|
||||
1880000000
|
||||
1870000000
|
||||
1860000000
|
||||
1850000000
|
||||
1840000000
|
||||
1830000000
|
||||
1820000000
|
||||
1810000000
|
||||
1800000000
|
||||
1500000000
|
||||
1490000000
|
||||
1480000000
|
||||
1470000000
|
||||
1460000000
|
||||
1450000000
|
||||
1440000000
|
||||
1430000000
|
||||
1420000000
|
||||
1410000000
|
||||
1400000000
|
||||
1390000000
|
||||
1380000000
|
||||
1370000000
|
||||
1360000000
|
||||
1350000000
|
||||
1340000000
|
||||
1330000000
|
||||
1320000000
|
||||
1310000000
|
||||
1300000000
|
||||
1290000000
|
||||
870000000
|
||||
860000000
|
||||
580000000
|
||||
570000000
|
||||
560000000
|
||||
550000000
|
||||
540000000
|
||||
530000000
|
||||
520000000
|
||||
510000000
|
||||
500000000
|
||||
490000000
|
||||
480000000
|
||||
470000000
|
||||
460000000
|
||||
450000000
|
||||
440000000
|
||||
430000000
|
||||
420000000
|
||||
410000000
|
||||
400000000
|
||||
40000000
|
||||
30000000
|
||||
20000000
|
||||
10000000
|
||||
5000000
|
||||
0
|
||||
317
netease_music/sql/export/distribution_range/user.txt
Normal file
317
netease_music/sql/export/distribution_range/user.txt
Normal file
@@ -0,0 +1,317 @@
|
||||
8080000000
|
||||
8070000000
|
||||
8060000000
|
||||
8050000000
|
||||
8040000000
|
||||
8030000000
|
||||
8020000000
|
||||
8010000000
|
||||
8000000000
|
||||
7990000000
|
||||
7980000000
|
||||
7970000000
|
||||
7960000000
|
||||
7950000000
|
||||
7940000000
|
||||
7930000000
|
||||
7920000000
|
||||
7910000000
|
||||
7900000000
|
||||
7890000000
|
||||
7880000000
|
||||
7870000000
|
||||
7860000000
|
||||
7850000000
|
||||
7840000000
|
||||
7830000000
|
||||
7820000000
|
||||
7810000000
|
||||
7800000000
|
||||
7790000000
|
||||
6490000000
|
||||
6480000000
|
||||
6470000000
|
||||
6460000000
|
||||
6450000000
|
||||
6440000000
|
||||
6430000000
|
||||
6420000000
|
||||
6410000000
|
||||
6400000000
|
||||
6390000000
|
||||
6380000000
|
||||
6370000000
|
||||
6360000000
|
||||
6350000000
|
||||
6340000000
|
||||
6330000000
|
||||
6320000000
|
||||
6310000000
|
||||
6300000000
|
||||
6290000000
|
||||
6280000000
|
||||
6270000000
|
||||
5890000000
|
||||
5220000000
|
||||
5210000000
|
||||
5200000000
|
||||
5190000000
|
||||
5180000000
|
||||
5170000000
|
||||
5160000000
|
||||
5150000000
|
||||
5140000000
|
||||
5130000000
|
||||
5120000000
|
||||
5110000000
|
||||
5100000000
|
||||
5090000000
|
||||
5080000000
|
||||
5070000000
|
||||
5060000000
|
||||
5050000000
|
||||
5040000000
|
||||
5030000000
|
||||
5020000000
|
||||
5010000000
|
||||
5000000000
|
||||
4990000000
|
||||
4980000000
|
||||
4970000000
|
||||
4960000000
|
||||
4950000000
|
||||
4940000000
|
||||
4930000000
|
||||
4920000000
|
||||
4910000000
|
||||
4900000000
|
||||
4890000000
|
||||
4880000000
|
||||
4870000000
|
||||
4060000000
|
||||
4050000000
|
||||
4040000000
|
||||
4030000000
|
||||
4020000000
|
||||
4010000000
|
||||
4000000000
|
||||
3990000000
|
||||
3980000000
|
||||
3970000000
|
||||
3960000000
|
||||
3950000000
|
||||
3940000000
|
||||
3930000000
|
||||
3920000000
|
||||
3910000000
|
||||
3900000000
|
||||
3890000000
|
||||
3880000000
|
||||
3870000000
|
||||
3860000000
|
||||
3850000000
|
||||
3840000000
|
||||
3830000000
|
||||
3820000000
|
||||
3810000000
|
||||
3800000000
|
||||
3790000000
|
||||
3780000000
|
||||
3770000000
|
||||
3760000000
|
||||
3750000000
|
||||
3740000000
|
||||
3730000000
|
||||
3720000000
|
||||
3710000000
|
||||
3700000000
|
||||
3690000000
|
||||
3680000000
|
||||
3670000000
|
||||
3660000000
|
||||
3650000000
|
||||
3640000000
|
||||
3630000000
|
||||
3620000000
|
||||
3610000000
|
||||
3600000000
|
||||
3590000000
|
||||
3580000000
|
||||
3570000000
|
||||
3560000000
|
||||
3550000000
|
||||
3540000000
|
||||
3530000000
|
||||
3520000000
|
||||
3510000000
|
||||
3500000000
|
||||
3490000000
|
||||
3480000000
|
||||
3470000000
|
||||
3460000000
|
||||
3450000000
|
||||
3440000000
|
||||
3430000000
|
||||
3420000000
|
||||
3410000000
|
||||
3400000000
|
||||
3390000000
|
||||
3380000000
|
||||
3370000000
|
||||
3360000000
|
||||
3350000000
|
||||
3340000000
|
||||
3330000000
|
||||
3320000000
|
||||
3310000000
|
||||
3300000000
|
||||
3290000000
|
||||
3280000000
|
||||
3270000000
|
||||
3260000000
|
||||
3250000000
|
||||
3240000000
|
||||
3230000000
|
||||
3220000000
|
||||
2140000000
|
||||
2130000000
|
||||
2120000000
|
||||
2110000000
|
||||
2100000000
|
||||
2090000000
|
||||
2080000000
|
||||
2070000000
|
||||
2060000000
|
||||
2050000000
|
||||
2040000000
|
||||
2030000000
|
||||
2020000000
|
||||
2010000000
|
||||
2000000000
|
||||
1990000000
|
||||
1980000000
|
||||
1970000000
|
||||
1960000000
|
||||
1950000000
|
||||
1940000000
|
||||
1930000000
|
||||
1920000000
|
||||
1910000000
|
||||
1900000000
|
||||
1890000000
|
||||
1880000000
|
||||
1870000000
|
||||
1860000000
|
||||
1850000000
|
||||
1840000000
|
||||
1830000000
|
||||
1820000000
|
||||
1810000000
|
||||
1800000000
|
||||
1790000000
|
||||
1780000000
|
||||
1770000000
|
||||
1760000000
|
||||
1750000000
|
||||
1740000000
|
||||
1730000000
|
||||
1720000000
|
||||
1710000000
|
||||
1700000000
|
||||
1690000000
|
||||
1680000000
|
||||
1670000000
|
||||
1660000000
|
||||
1650000000
|
||||
1640000000
|
||||
1630000000
|
||||
1620000000
|
||||
1610000000
|
||||
1600000000
|
||||
1590000000
|
||||
1580000000
|
||||
1570000000
|
||||
1560000000
|
||||
1550000000
|
||||
1540000000
|
||||
1530000000
|
||||
1520000000
|
||||
1510000000
|
||||
1500000000
|
||||
1490000000
|
||||
1480000000
|
||||
1470000000
|
||||
1460000000
|
||||
1450000000
|
||||
1440000000
|
||||
1430000000
|
||||
1420000000
|
||||
1410000000
|
||||
1400000000
|
||||
1390000000
|
||||
1380000000
|
||||
1370000000
|
||||
1360000000
|
||||
1350000000
|
||||
1340000000
|
||||
1330000000
|
||||
1320000000
|
||||
1310000000
|
||||
1300000000
|
||||
1290000000
|
||||
650000000
|
||||
640000000
|
||||
630000000
|
||||
620000000
|
||||
610000000
|
||||
600000000
|
||||
590000000
|
||||
580000000
|
||||
570000000
|
||||
560000000
|
||||
550000000
|
||||
540000000
|
||||
530000000
|
||||
520000000
|
||||
510000000
|
||||
500000000
|
||||
490000000
|
||||
480000000
|
||||
470000000
|
||||
460000000
|
||||
450000000
|
||||
440000000
|
||||
430000000
|
||||
420000000
|
||||
410000000
|
||||
400000000
|
||||
390000000
|
||||
380000000
|
||||
370000000
|
||||
360000000
|
||||
350000000
|
||||
340000000
|
||||
330000000
|
||||
320000000
|
||||
310000000
|
||||
300000000
|
||||
290000000
|
||||
280000000
|
||||
270000000
|
||||
260000000
|
||||
250000000
|
||||
140000000
|
||||
130000000
|
||||
120000000
|
||||
110000000
|
||||
100000000
|
||||
90000000
|
||||
80000000
|
||||
70000000
|
||||
60000000
|
||||
50000000
|
||||
40000000
|
||||
30000000
|
||||
20000000
|
||||
10000000
|
||||
0
|
||||
83
netease_music/sql/export/export_sql_generator.js
Normal file
83
netease_music/sql/export/export_sql_generator.js
Normal file
@@ -0,0 +1,83 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
|
||||
const absPath = `D:/sql_export`;
|
||||
|
||||
// 数字转成字符串，同时在前面填充
|
||||
function fill(num, fillers, length) {
|
||||
var result = `${num}`;
|
||||
if (result.length < length)
|
||||
result = new Array(length - result.length + 1).join(fillers) + result;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
const export_gap = 50000000;
|
||||
const partition_gap = 50000000;
|
||||
const table = "comment_1024";
|
||||
const index = "comment_id";
|
||||
const exportTablePrefix = "comment_export_";
|
||||
|
||||
// 数字转成字符串，同时在前面填充
|
||||
function fill(num, fillers, length) {
|
||||
var result = `${num}`;
|
||||
if (result.length < length)
|
||||
result = new Array(length - result.length + 1).join(fillers) + result;
|
||||
return result;
|
||||
}
|
||||
|
||||
let sqlArr1 = [];
|
||||
let sqlArr2 = [];
|
||||
let sqlArr3 = [];
|
||||
for (let i = 200; i < 300; i++) {
|
||||
// for (let i = 100; i < 200; i++) {
|
||||
let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
|
||||
|
||||
let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
|
||||
sqlArr1.push(sql_create_table);
|
||||
|
||||
let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
|
||||
sqlArr2.push(sql_delete_rows);
|
||||
|
||||
let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
|
||||
sqlArr3.push(sql_partition);
|
||||
}
|
||||
sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
|
||||
|
||||
// 建表
|
||||
// console.log(sqlArr1.join('\n'));
|
||||
|
||||
// 删除原表数据
|
||||
// console.log(sqlArr2.join('\n'));
|
||||
|
||||
// 新创建表的分区
|
||||
// console.log(`partition (\n${sqlArr3.join(',\n')}\n)`);
|
||||
|
||||
// #############################################
|
||||
|
||||
// 删除表
|
||||
// for (let i = 107; i < 200; i++) {
|
||||
// console.log(`DROP TABLE IF EXISTS comment_export_${fill(i, '0', 4)};`);
|
||||
// }
|
||||
|
||||
// #############################################
|
||||
|
||||
// 使用 mysqldump 分块导出数据表
|
||||
let rangeTxtName = "album"; // 分布区间 "song" "album" "artist"
|
||||
const dumpTable = "album"; // "comment_progress";
|
||||
const fieldName = `album_id`;
|
||||
var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim());
|
||||
// console.log(a);
|
||||
let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`];
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
let where;
|
||||
if (a[i + 1]) {
|
||||
where = `${fieldName}>=${a[i]} and ${fieldName}<${a[i + 1]}`;
|
||||
} else {
|
||||
where = `${fieldName}>=${a[i]}`;
|
||||
}
|
||||
outputArr.push(`mysqldump neteasemusic -hrm-bp18qrc78dj7vd3newo.rwlb.rds.aliyuncs.com -uroot -pOj13EzoppxXvMmjPKh --tables ${dumpTable} --where="${where}" --skip-add-drop-table --set-gtid-purged=OFF > ${absPath}/${dumpTable}_${fill(i, '0', 4)}.sql`);
|
||||
}
|
||||
outputArr.push("echo done.");
|
||||
console.log(outputArr.join('\n\n'));
|
||||
Reference in New Issue
Block a user