coder-xiaomo/tools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity GitHub Gitee

添加sql导入脚本生成工具

Browse Source
This commit is contained in:
程序员小墨 2022-10-26 20:44:07 +08:00
parent 66d5f89b02
commit 19e69f9bfd
3 changed files with 50 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
netease_music/sql/export/export_sql_generator.js
View File

@ -1,7 +1,6 @@
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const absPath = `D:/sql_export`; const absPath = `D:/sql_export`;
// 数字转成字符串，同时在前面填充 // 数字转成字符串，同时在前面填充
@ -12,38 +11,31 @@ function fill(num, fillers, length) {
return result; return result;
} }
// #############################################
const export_gap = 50000000; // const export_gap = 50000000;
const partition_gap = 50000000; // const partition_gap = 50000000;
const table = "comment_1024"; // const table = "comment_1024";
const index = "comment_id"; // const index = "comment_id";
const exportTablePrefix = "comment_export_"; // const exportTablePrefix = "comment_export_";
// 数字转成字符串，同时在前面填充 // let sqlArr1 = [];
function fill(num, fillers, length) { // let sqlArr2 = [];
var result = `${num}`; // let sqlArr3 = [];
if (result.length < length) // for (let i = 200; i < 300; i++) {
result = new Array(length - result.length + 1).join(fillers) + result; // // for (let i = 100; i < 200; i++) {
return result; // let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
}
let sqlArr1 = []; // let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`;
let sqlArr2 = []; // sqlArr1.push(sql_create_table);
let sqlArr3 = [];
for (let i = 200; i < 300; i++) {
// for (let i = 100; i < 200; i++) {
let where = `${index} >= ${fill(i * export_gap, ' ', 12)} and ${index} < ${fill((i + 1) * export_gap, ' ', 12)}`;
let sql_create_table = `create table ${exportTablePrefix}${fill(i, '0', 4)} select * from ${table} where ${where};`; // let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`;
sqlArr1.push(sql_create_table); // sqlArr2.push(sql_delete_rows);
let sql_delete_rows = `DELETE FROM ${table} WHERE ${where};`; // let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`;
sqlArr2.push(sql_delete_rows); // sqlArr3.push(sql_partition);
// }
let sql_partition = ` PARTITION p${fill(i, '0', 4)} VALUES LESS THAN (${fill((i + 1) * partition_gap, ' ', 12)})`; // sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
sqlArr3.push(sql_partition);
}
sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// 建表 // 建表
// console.log(sqlArr1.join('\n')); // console.log(sqlArr1.join('\n'));
@ -64,9 +56,9 @@ sqlArr3.push(` PARTITION p_max VALUES LESS THAN MAXVALUE`);
// ############################################# // #############################################
// 使用 mysqldump 分块导出数据表 // 使用 mysqldump 分块导出数据表
let rangeTxtName = "album"; // 分布区间 "song" "album" "artist" let rangeTxtName = "song"; // 分布区间 "song" "album" "artist" "user"
const dumpTable = "album"; // "comment_progress"; const dumpTable = "lyric"; // "comment_progress";
const fieldName = `album_id`; const fieldName = `song_id`;
var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim()); var a = fs.readFileSync(path.join(__dirname, `distribution_range/${rangeTxtName}.txt`), "utf-8").trim().split("\n").reverse().map(i => i.trim());
// console.log(a); // console.log(a);
let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`]; let outputArr = [`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`];

26
netease_music/sql/import/import_sql_generator.js Normal file
View File

@ -0,0 +1,26 @@
const absPath = `D:/sql_export`;
// 数字转成字符串，同时在前面填充
function fill(num, fillers, length) {
var result = `${num}`;
if (result.length < length)
result = new Array(length - result.length + 1).join(fillers) + result;
return result;
}
// 使用 mysql 导入数据表
let outputArr = [
[
`@echo off`, `D:`, `cd D:/Program/Development/Environment/phpstudy_pro/Extensions/MySQL8.0.12/bin`,
`mysql -hlocalhost -uroot -proot neteasemusic`
].join('\n'),
`use neteasemusic;`
];
let firstIndex = 0;
let lastIndex = 115;
for (let i = firstIndex; i <= lastIndex; i++) {
outputArr.push(`source ${absPath}/comment/comment_${fill(i, '0', 4)}.sql`);
}
outputArr.push("echo done.");
console.log(outputArr.join('\n'));

1
netease_music/todo.txt
View File

@ -31,6 +31,7 @@ node index --utils playlist
后期： 后期：
删除song_playlist_relation表中rcmd_reason字段（全是空字符串）
歌单定时更新（rel表中添加一个del字段，先将歌单下面的全部置为删除状态，再插入的时候把已有歌曲的标记重新修改为正常状态） 歌单定时更新（rel表中添加一个del字段，先将歌单下面的全部置为删除状态，再插入的时候把已有歌曲的标记重新修改为正常状态）