admin/gr_bi_java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Signed-off-by: chy <chy@163.com>

Browse Source
This commit is contained in:
chy
2026-02-02 23:31:39 +08:00
commit f6d2459f1f
1499 changed files with 289491 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
lidee-common/lidee-boot-archiver/pom.xml Normal file
View File

@@ -0,0 +1,61 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>top.lidee.taie</groupId>
<artifactId>lidee-common</artifactId>
<version>${revision}</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>top.lidee.taie</groupId>
<artifactId>lidee-boot-archiver</artifactId>
<name>lidee-boot-archiver</name>
<description>lidee-boot-archiver</description>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jdbc</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
</plugin>
</plugins>
</build>
</project>

162
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/config/ArchiverProperties.java Normal file
View File

@@ -0,0 +1,162 @@
package top.lidee.taie.archiver.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.ArrayList;
import java.util.List;
/**
* 归档组件配置项
*
* @author: Devli
* @since 2021/2/3 14:16
*/
@ConfigurationProperties(prefix = ArchiverProperties.COMPONENT_PREFIX + ArchiverProperties.COMPONENT_NAME)
public class ArchiverProperties {
public final static String COMPONENT_PREFIX="spring.lidee.subscribes.";
/**
* 组件名称
*/
public final static String COMPONENT_NAME = "archiver";
private boolean enabled = true;
/** 归档触发的定时器 */
private String archiveScheduledCron;
/** 将n天前的数据，移动到归档表，tables配置项中策略优先加载 */
private Integer maxDaysBeforeArchive;
/** 归档表中，已归档的数据保留期限，超过期限的数据将删除。tables配置项中策略优先加载 */
private Integer maxDaysBeforeDelete;
/** 删除历史数据表，是否需要存入本地文件 */
private boolean historicalBackup = false;
/** 生效要归档的库名 historicalBackup为true生效*/
private String dbName;
/** dump 归档host historicalBackup为true生效*/
private String dumpHost;
/** dump 归档port historicalBackup为true生效*/
private String dumpPort;
/** dump 归档用户 historicalBackup为true生效*/
private String dumpUser;
/** dump 归档密码 historicalBackup为true生效*/
private String dumpPasswd;
/** dump 归档本地路径 historicalBackup为true生效*/
private String dumpFile;
/** 限制执行节点ip */
private String executeIp;
/** 归档表配置 */
private List<ArchiverTable> tables = new ArrayList<ArchiverTable>();
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public boolean isHistoricalBackup() {
return historicalBackup;
}
public void setHistoricalBackup(boolean historicalBackup) {
this.historicalBackup = historicalBackup;
}
public String getDbName() {
return dbName;
}
public void setDbName(String dbName) {
this.dbName = dbName;
}
public String getDumpHost() {
return dumpHost;
}
public void setDumpHost(String dumpHost) {
this.dumpHost = dumpHost;
}
public String getDumpPort() {
return dumpPort;
}
public void setDumpPort(String dumpPort) {
this.dumpPort = dumpPort;
}
public String getDumpUser() {
return dumpUser;
}
public void setDumpUser(String dumpUser) {
this.dumpUser = dumpUser;
}
public String getDumpPasswd() {
return dumpPasswd;
}
public void setDumpPasswd(String dumpPasswd) {
this.dumpPasswd = dumpPasswd;
}
public String getDumpFile() {
return dumpFile;
}
public void setDumpFile(String dumpFile) {
this.dumpFile = dumpFile;
}
public String getArchiveScheduledCron() {
return archiveScheduledCron;
}
public void setArchiveScheduledCron(String archiveScheduledCron) {
this.archiveScheduledCron = archiveScheduledCron;
}
public Integer getMaxDaysBeforeArchive() {
return maxDaysBeforeArchive;
}
public void setMaxDaysBeforeArchive(Integer maxDaysBeforeArchive) {
this.maxDaysBeforeArchive = maxDaysBeforeArchive;
}
public Integer getMaxDaysBeforeDelete() {
return maxDaysBeforeDelete;
}
public void setMaxDaysBeforeDelete(Integer maxDaysBeforeDelete) {
this.maxDaysBeforeDelete = maxDaysBeforeDelete;
}
public List<ArchiverTable> getTables() {
return tables;
}
public void setTables(List<ArchiverTable> tables) {
this.tables = tables;
}
public String getExecuteIp() {
return executeIp;
}
public void setExecuteIp(String executeIp) {
this.executeIp = executeIp;
}
}

133
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/config/ArchiverTable.java Normal file
View File

@@ -0,0 +1,133 @@
package top.lidee.taie.archiver.config;
import java.util.List;
/**
* 归档组件配置项
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class ArchiverTable {
/** 要归档的表名 */
private String tablename;
/** 归档依据的时间列名 */
private String timefield;
/** 归档多久之前的数据，将n天前的数据，移动到归档表 */
private Integer maxDaysBeforeArchive;
/** 删除多久之前的历史数据，归档表中，已归档的数据保留期限，超过期限的数据将删除。*/
private Integer maxDaysBeforeDelete;
/** 要归档的库名 */
private String dbName;
/** dump 归档host */
private String dumpHost;
/** dump 归档port */
private String dumpPort;
/** dump 归档用户 */
private String dumpUser;
/** dump 归档密码 */
private String dumpPasswd;
/** dump 归档本地路径 */
private String dumpFile;
/** dump 归档本地路径 */
private List<String> dumpTableList;
public String getDbName() {
return dbName;
}
public void setDbName(String dbName) {
this.dbName = dbName;
}
public List<String> getDumpTableList() {
return dumpTableList;
}
public void setDumpTableList(List<String> dumpTableList) {
this.dumpTableList = dumpTableList;
}
public String getDumpHost() {
return dumpHost;
}
public void setDumpHost(String dumpHost) {
this.dumpHost = dumpHost;
}
public String getDumpPort() {
return dumpPort;
}
public void setDumpPort(String dumpPort) {
this.dumpPort = dumpPort;
}
public String getDumpUser() {
return dumpUser;
}
public void setDumpUser(String dumpUser) {
this.dumpUser = dumpUser;
}
public String getDumpPasswd() {
return dumpPasswd;
}
public void setDumpPasswd(String dumpPasswd) {
this.dumpPasswd = dumpPasswd;
}
public String getDumpFile() {
return dumpFile;
}
public void setDumpFile(String dumpFile) {
this.dumpFile = dumpFile;
}
public String getTablename() {
return tablename;
}
public void setTablename(String tablename) {
this.tablename = tablename;
}
public String getTimefield() {
return timefield;
}
public void setTimefield(String timefield) {
this.timefield = timefield;
}
public Integer getMaxDaysBeforeArchive() {
return maxDaysBeforeArchive;
}
public void setMaxDaysBeforeArchive(Integer maxDaysBeforeArchive) {
this.maxDaysBeforeArchive = maxDaysBeforeArchive;
}
public Integer getMaxDaysBeforeDelete() {
return maxDaysBeforeDelete;
}
public void setMaxDaysBeforeDelete(Integer maxDaysBeforeDelete) {
this.maxDaysBeforeDelete = maxDaysBeforeDelete;
}
}

71
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/config/AutoConfiguration.java Normal file
View File

@@ -0,0 +1,71 @@
package top.lidee.taie.archiver.config;
import top.lidee.taie.archiver.service.ArchiverService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.SchedulingConfigurer;
import org.springframework.scheduling.config.ScheduledTaskRegistrar;
import org.springframework.scheduling.config.TriggerTask;
import org.springframework.scheduling.support.CronTrigger;
import org.springframework.util.StringUtils;
import java.net.InetAddress;
/**
* springboot自动装配类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
@Configuration
@EnableScheduling
@EnableConfigurationProperties(ArchiverProperties.class)
@ComponentScan("top.lidee.taie.archiver")
@ConditionalOnProperty(name= ArchiverProperties.COMPONENT_PREFIX + ArchiverProperties.COMPONENT_NAME + ".enabled", havingValue="true")
public class AutoConfiguration implements SchedulingConfigurer {
private static Logger logger = LoggerFactory.getLogger(AutoConfiguration.class);
@Autowired
private ArchiverProperties archiverProperties;
@Autowired
private ArchiverService archiverService;
public AutoConfiguration() {
logger.info("spring lidee subscribes archiver is actived");
}
@Override
public void configureTasks(ScheduledTaskRegistrar scheduledTaskRegistrar) {
if (archiverProperties.getArchiveScheduledCron() != null && !StringUtils.isEmpty(archiverProperties.getExecuteIp())) {
//限制pi
try {
InetAddress addr = InetAddress.getLocalHost();
String localIp = addr.getHostAddress();
logger.info("Scheduled Task. localIp:{}, executeIp：{}", localIp, archiverProperties.getExecuteIp());
if(!archiverProperties.getExecuteIp().equals(localIp)) {
logger.info("No need to execute scheduled...");
return;
}
} catch (Exception e) {
logger.error(e.getMessage());
return;
}
}
scheduledTaskRegistrar.addTriggerTask(new TriggerTask(() -> archiverService.doArchiveTable(), triggerContext -> {
// 定义定时器的Cron间隔，默认每月2号，凌晨3:0:0执行归档
String cron = "0 0 3 2 */1 ?";
if(archiverProperties.getArchiveScheduledCron() != null && archiverProperties.getArchiveScheduledCron().trim().length() > 5){
cron = archiverProperties.getArchiveScheduledCron().trim();
}
//调用执行器
return new CronTrigger(cron).nextExecutionTime(triggerContext);
}));
}
}

57
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/consant/DBType.java Normal file
View File

@@ -0,0 +1,57 @@
package top.lidee.taie.archiver.consant;
/**
* 归档组件支持的数据库类型
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public enum DBType {
MYSQL("mysql", "MySql数据库"),
MARIADB("mariadb", "MariaDB数据库"),
ORACLE("oracle", "Oracle数据库"),
SQL_SERVER("sqlserver", "SQLServer数据库"),
//以下还未支持，后期逐步扩展
DB2("db2", "DB2数据库"),
H2("h2", "H2数据库"),
HSQL("hsql", "HSQL数据库"),
SQLITE("sqlite", "SQLite数据库"),
POSTGRE_SQL("postgresql", "Postgre数据库"),
DM("dm", "达梦数据库"),
XU_GU("xugu", "虚谷数据库"),
KINGBASE_ES("kingbasees", "人大金仓数据库"),
PHOENIX("phoenix", "Phoenix HBase数据库"),
GAUSS("zenith", "Gauss 数据库"),
OTHER("other", "其他数据库");
private final String db;
private final String desc;
DBType(final String db, final String desc) {
this.db = db;
this.desc = desc;
}
public static DBType getDbType(String dbType) {
DBType[] dbTypes = values();
int size = dbTypes.length;
for(int i = 0; i < size; ++i) {
DBType type = dbTypes[i];
if (type.db.equalsIgnoreCase(dbType)) {
return type;
}
}
return OTHER;
}
public String getDb() {
return db;
}
public String getDesc() {
return desc;
}
}

240
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/service/ArchiverService.java Normal file
View File

@@ -0,0 +1,240 @@
package top.lidee.taie.archiver.service;
import top.lidee.taie.archiver.config.ArchiverProperties;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.utils.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.stereotype.Service;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import org.springframework.util.StringUtils;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
/**
* 归档业务实现类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
@Service
public class ArchiverService implements IArchiverService{
private static Logger logger = LoggerFactory.getLogger(ArchiverService.class);
@Autowired
private ArchiverProperties archiverProperties;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private DataSourceTransactionManager dataSourceTransactionManager;
@Autowired
private IDBOperatorService dbOperatorService;
@Override
public List<ArchiverTable> validNeedArchiverTable(){
//配置项检查
List<ArchiverTable> needArchiveTables = new ArrayList<ArchiverTable>();
for(ArchiverTable archiverTable: archiverProperties.getTables()){
//判断表名是否为空
if(archiverTable.getTablename() == null || archiverTable.getTablename().trim().equals("")){
logger.warn("lidee archive detect a blank table, please check your configuration");
continue;
}
String tableName = archiverTable.getTablename().trim();
//判断列名是否为空
if(archiverTable.getTimefield() == null || archiverTable.getTimefield().trim().equals("")){
logger.warn("lidee archive detect a blank field name for table [{}], please check your configuration", tableName);
continue;
}
String fieldName = archiverTable.getTimefield().trim();
//判断表名、列名是否存在
boolean isExist = dbOperatorService.existTableField(tableName, fieldName);
if(!isExist){
logger.warn("lidee archive detect field [{}] not exist in table [{}], please check your configuration", fieldName, tableName);
continue;
}
//判断配置项，归档多久之前的数据
if(archiverTable.getMaxDaysBeforeArchive() == null || archiverTable.getMaxDaysBeforeArchive().intValue() <= 0){
if(archiverProperties.getMaxDaysBeforeArchive() != null && archiverProperties.getMaxDaysBeforeArchive().intValue() > 0){
archiverTable.setMaxDaysBeforeArchive(archiverProperties.getMaxDaysBeforeArchive());
}else{
logger.warn("lidee archive task will be disable for table [{}], because detect maxDaysBeforeArchive less than 0.", tableName);
continue;
}
}
//判断配置项，删除多久之前的历史数据
if(archiverTable.getMaxDaysBeforeDelete() == null || archiverTable.getMaxDaysBeforeDelete().intValue() <= 0){
if(archiverProperties.getMaxDaysBeforeDelete() != null && archiverProperties.getMaxDaysBeforeDelete().intValue() > 0){
archiverTable.setMaxDaysBeforeDelete(archiverProperties.getMaxDaysBeforeDelete());
}
}
if(archiverTable.getMaxDaysBeforeDelete() != null && archiverTable.getMaxDaysBeforeDelete().intValue() <= archiverTable.getMaxDaysBeforeArchive().intValue()){
archiverTable.setMaxDaysBeforeDelete(null);
logger.warn("lidee delete task will be disable for table [{}], because maxDaysBeforeDelete {} less than maxDaysBeforeArchive {}",
tableName, archiverTable.getMaxDaysBeforeDelete(), archiverTable.getMaxDaysBeforeArchive());
}
if (StringUtils.isEmpty(archiverTable.getDbName())) {
archiverTable.setDbName(archiverProperties.getDbName());
}
//删除历史数据表，是否需要存入本地文件
if (archiverProperties.isHistoricalBackup()) {
if (StringUtils.isEmpty(archiverTable.getDumpHost())) {
archiverTable.setDumpHost(archiverProperties.getDumpHost());
}
if (StringUtils.isEmpty(archiverTable.getDumpPort())) {
archiverTable.setDumpPort(archiverProperties.getDumpPort());
}
if (StringUtils.isEmpty(archiverTable.getDumpUser())) {
archiverTable.setDumpUser(archiverProperties.getDumpUser());
}
if (StringUtils.isEmpty(archiverTable.getDumpPasswd())) {
archiverTable.setDumpPasswd(archiverProperties.getDumpPasswd());
}
if (StringUtils.isEmpty(archiverTable.getDumpFile())) {
archiverTable.setDumpFile(archiverProperties.getDumpFile());
}
}
needArchiveTables.add(archiverTable);
}
return needArchiveTables;
}
@Override
public void archiveTable(ArchiverTable archiverTable){
String tablename = archiverTable.getTablename();
String timefield = archiverTable.getTimefield();
Integer maxDaysBeforeArchive = archiverTable.getMaxDaysBeforeArchive();
Integer maxDaysBeforeDelete = archiverTable.getMaxDaysBeforeDelete();
//查询出历史数据，有哪些月份
//select date_format(request_time,'%Y-%m') from t_log group by date_format(request_time,'%Y-%m');
List<String> monthList = dbOperatorService.getCrossMonthList(tablename, timefield, maxDaysBeforeArchive);
for(String monthStr: monthList){
DefaultTransactionDefinition def = new DefaultTransactionDefinition();
def.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus status = dataSourceTransactionManager.getTransaction(def);
//创建归档表
String archiveTableName = dbOperatorService.createArchiverTableIfNotExist(tablename, monthStr,archiverTable);
//获取某月第一天和最后一天
String startDay = String.format("%s-01", monthStr);
String endDay = DateUtil.formatDate(DateUtil.addMonth(startDay, 1));
String archiveSQl = String.format("insert into %s select * from %s where %s>='%s' and %s<'%s'",
archiveTableName, tablename, timefield, startDay, timefield, endDay);
String deletesQL = String.format("delete from %s where %s>='%s' and %s<'%s'",
tablename, timefield, startDay, timefield, endDay);
logger.info("archive:insert-sql:{}",archiveSQl);
logger.info("archive:delete-sql:{}",deletesQL);
int archiveCount = jdbcTemplate.update(archiveSQl);
int deleteCount = jdbcTemplate.update(deletesQL);
if(archiveCount != deleteCount){
dataSourceTransactionManager.rollback(status);
}
dataSourceTransactionManager.commit(status);
logger.info("archive table {} -> {} success, row count={}", tablename, archiveTableName, archiveCount);
}
}
@Override
public void dropOldArchive(ArchiverTable archiverTable) {
String tablename = archiverTable.getTablename();
String dbName = archiverTable.getDbName();
Integer maxDaysBeforeDelete = archiverTable.getMaxDaysBeforeDelete();
//根据tablename，扫描所有归档表
List<String> tableNameList = dbOperatorService.scanArchiver(tablename, dbName);
//根据maxDaysBeforeDelete判断归档表是否已经可以删除
List<String> needDelOldArchiveTableList = needDelOldArchive(tableNameList, maxDaysBeforeDelete);
if (null == needDelOldArchiveTableList || needDelOldArchiveTableList.size() < 1) {
return;
}
archiverTable.setDumpTableList(needDelOldArchiveTableList);
//如果已过期，删除前，先dump到服务器
if (archiverProperties.isHistoricalBackup()) {
dbOperatorService.executeDump(archiverTable);
}
//dump完成后drop table
dbOperatorService.dropTable(archiverTable);
logger.info("drop old archive table {} success", tablename);
}
public List<String> needDelOldArchive(List<String> tableNameList, Integer maxDaysBeforeDelete) {
List<String> result = new ArrayList<>();
if(maxDaysBeforeDelete == null){
return result;
}
Date dateBefore = DateUtil.getDateBefore(new Date(), maxDaysBeforeDelete);
if (null != tableNameList && tableNameList.size() > 0) {
tableNameList.forEach(tableName -> {
//tableName_yyyyMM
String month = tableName.substring(tableName.lastIndexOf("_") + 1);
Date monthDate = DateUtil.parse(month, "yyyyMM");
if (null != monthDate) {
if (DateUtil.before(monthDate, dateBefore)) {
//在历史归档期限之前
result.add(tableName);
}
}
});
}
return result;
}
@Override
public void doArchiveTable(){
//判断归档开关是否打开
logger.info("loading lidee archiver");
if(archiverProperties == null || archiverProperties.isEnabled() == false || archiverProperties.getTables() == null || archiverProperties.getTables().size() == 0){
logger.warn("lidee archive disabled,it's configuration item enabled = false or table list is empty");
return;
}
//开始时间
LocalDateTime startTime = LocalDateTime.now();
//检查配置项中配置的归档表，校验表名列名是否正确
List<ArchiverTable> archiverTableList = validNeedArchiverTable();
//启动归档任务 删除旧归档
for (ArchiverTable archiverTable: archiverTableList) {
try {
archiveTable(archiverTable);
dropOldArchive(archiverTable);
}catch (Exception ex){
logger.error("lidee archive err:{}",archiverTable.getTablename(),ex);
}
}
//结束时间
LocalDateTime endTime = LocalDateTime.now();
double second = Duration.between(startTime, endTime).toMillis()/1000.0;
logger.info("lidee archive finished,table:{},time cost:{} seconds",
archiverTableList.stream().map(ArchiverTable::getTablename)
.collect(Collectors.joining(",")), second);
}
}

231
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/service/DBOperatorService.java Normal file
View File

@@ -0,0 +1,231 @@
package top.lidee.taie.archiver.service;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.consant.DBType;
import top.lidee.taie.archiver.sqlbuilder.ISQLBuilder;
import top.lidee.taie.archiver.sqlbuilder.SQLBuilderFactory;
import top.lidee.taie.archiver.utils.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.support.rowset.SqlRowSet;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;
/**
* 数据库操作实现类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
@Service
public class DBOperatorService implements IDBOperatorService {
private static DBType dbType;
private static Logger logger = LoggerFactory.getLogger(DBOperatorService.class);
@Autowired
private JdbcTemplate jdbcTemplate;
private DBType getDBType() throws SQLException {
if(dbType != null){
return dbType;
}
String databaseProductName=jdbcTemplate.getDataSource().getConnection().getMetaData().getDatabaseProductName();
if(databaseProductName == null || databaseProductName.trim().length() == 0){
throw new RuntimeException(String.format("%s database not supported", databaseProductName));
}
dbType=DBType.getDbType(databaseProductName);
return Optional.ofNullable(dbType).orElseThrow(()->
new RuntimeException(String.format("%s database not supported", databaseProductName))
);
}
private ISQLBuilder getSQLBuilder(){
try{
return SQLBuilderFactory.getSQLBuilder(getDBType());
}catch (Exception e){
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
@Override
public boolean existTable(String tableName) {
try{
String sql = getSQLBuilder().existTable(tableName);
logger.info("exist-table-sql={}",sql);
jdbcTemplate.queryForRowSet(sql);
return true;
}catch (Exception e){
logger.error("exist-err:{}",tableName,e);
return false;
}
}
@Override
public boolean existTableField(String tableName, String fieldName) {
try{
String sql = getSQLBuilder().existTableField(tableName, fieldName);
logger.info("get-table-metadata-sql={}",sql);
SqlRowSet sqlRowSet = jdbcTemplate.queryForRowSet(sql);
boolean existField = false;
while (sqlRowSet.next()){
String fileName = sqlRowSet.getString("Field");
if(fileName.toLowerCase().trim().equals(fieldName.trim())){
existField = true;
break;
}
}
return existField;
}catch (Exception e){
logger.error("exist-err:{}:{}",tableName,fieldName,e);
return false;
}
}
@Override
public Date getNow() {
String sql = getSQLBuilder().getNow();
Map<String, Object> map = jdbcTemplate.queryForMap(sql);
Date now = (Date)map.get("now");
return null;
}
@Override
public Long countRows(String sourceTableName) {
return null;
}
@Override
public List<String> getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive) {
String sql = getSQLBuilder().getCrossMonthList(tablename, timefield, maxDaysBeforeArchive);
logger.info("get-month-sql={}",sql);
List<Map<String, Object>> list = jdbcTemplate.queryForList(sql);
//yyyy-MM
List<String> monthList = new ArrayList<String>();
if(list == null ||list.isEmpty()){
logger.warn("get-month-sql-ret-is-empty");
return monthList;
}
for(Map<String, Object> item: list){
String monthStr = (String) item.get("month");
monthList.add(monthStr);
}
logger.warn("get-month-sql-ret:{}",monthList);
return monthList;
}
@Override
public String createArchiverTableIfNotExist(String sourceTableName, String monthStr){
monthStr = monthStr.replace("-","");
String archiveTableName = String.format("%s_%s", sourceTableName, monthStr);
if(existTable(archiveTableName) == false){
String sql = getSQLBuilder().createArchiverTableIfNotExist(archiveTableName, sourceTableName);
logger.info("create-table-sql={}",sql);
jdbcTemplate.execute(sql);
}
return archiveTableName;
}
@Override
public String createArchiverTableIfNotExist(String tableName, String monthStr, ArchiverTable archiverTable) {
monthStr = monthStr.replace("-","");
// table_202312,table_202212,table_202112
List<String> archiveTables = scanArchiver(tableName,archiverTable.getDbName());
if(!CollectionUtils.isEmpty(archiveTables)) {
int curr = Integer.valueOf(monthStr);
// 归档表
List<String> list = archiveTables.stream().sorted().collect(Collectors.toList());
// 最新的归档表
String latest = list.get(list.size()-1);
latest = latest.replace(tableName, "");
latest = latest.replace("_","");
int lastArchive = Integer.valueOf(latest);
if(curr <=lastArchive ){
for(int i=0;i<list.size()-1;i++){
String m = list.get(i).replace(tableName,"").replace("_","");
// 历史数据 找到最合适的归档表
if(curr <= Integer.valueOf(m)){
return list.get(i);
}
}
// 最近的一个归档表
return list.get(list.size()-1);
}else{
// 没找到归档表，新建一个归档表
String fmt = "yyyyMM";
String fmtDay = "yyyyMMdd";
long nextArchive = DateUtil.addDays(DateUtil.parse(latest+"01",fmtDay),
archiverTable.getMaxDaysBeforeArchive()).getTime();
monthStr = DateUtil.format(new Date(nextArchive),fmt);
}
}
String archiveTableName = String.format("%s_%s", tableName, monthStr);
if(!existTable(archiveTableName)){
String sql = getSQLBuilder().createArchiverTableIfNotExist(archiveTableName, tableName);
logger.info("create-table-sql={}",sql);
jdbcTemplate.execute(sql);
}
return archiveTableName;
}
@Override
public boolean doArchiver(String sourceTableName, Date startTime, Date endTime) {
return false;
}
/**
* 扫描以表名为前缀的所有归档表
*
* @param sourceTableName
* @return list
*/
@Override
public List<String> scanArchiver(String sourceTableName, String dbName) {
String sql = getSQLBuilder().scanArchive(sourceTableName, dbName,"_20");
List<Map<String, Object>> maps = jdbcTemplate.queryForList(sql);
List<String> tableNameList = new ArrayList<String>();
maps.forEach(stringObjectMap -> {
String tableName = (String) stringObjectMap.get("table_name");
tableNameList.add(tableName);
});
logger.info("scan-table-sql={},tables:{}",sql,tableNameList);
return tableNameList;
}
/**
* 执行dump命令
*
* @param archiverTable
* @return
*/
@Override
public String executeDump(ArchiverTable archiverTable) {
return getSQLBuilder().executeDump(archiverTable);
}
/**
* drop 过期的数据库表
*
* @param archiverTable
* @return
*/
@Override
public String dropTable(ArchiverTable archiverTable) {
String sql = getSQLBuilder().dropTable(archiverTable);
logger.info("drop-table-sql={}",sql);
jdbcTemplate.execute(sql);
return null;
}
}

36
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/service/IArchiverService.java Normal file
View File

@@ -0,0 +1,36 @@
package top.lidee.taie.archiver.service;
import top.lidee.taie.archiver.config.ArchiverTable;
import java.util.List;
/**
* 归档业务接口定义类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public interface IArchiverService {
/**检查配置项中配置的归档表，校验表名列名是否正确
* @return 有效的List<ArchiverTable>
*/
List<ArchiverTable> validNeedArchiverTable();
/**归档一张表
* @param archiverTable
*/
void archiveTable(ArchiverTable archiverTable);
/** 删除旧的归档表
* @param archiverTable
*/
void dropOldArchive(ArchiverTable archiverTable);
/**
* 执行归档任务
*/
void doArchiveTable();
}

91
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/service/IDBOperatorService.java Normal file
View File

@@ -0,0 +1,91 @@
package top.lidee.taie.archiver.service;
import top.lidee.taie.archiver.config.ArchiverTable;
import java.util.Date;
import java.util.List;
/**
* 数据库操作接口定义类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public interface IDBOperatorService {
/** 判断表名是否存在
* @param tableName
* @return
*/
boolean existTable(String tableName);
/** 判断表中列名是否存在
* @param tableName
* @param fieldName
* @return
*/
boolean existTableField(String tableName, String fieldName);
/** 获取数据库当前时间
* @return
*/
Date getNow();
/** 统计有多少条数据要归档
* @param sourceTableName
* @return
*/
Long countRows(String sourceTableName);
/** 查询本次归档的任务，跨了哪几个月份
* @param tablename
* @param timefield
* @param maxDaysBeforeArchive
* @return
*/
List<String> getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive);
/** 创建归档表
* @param sourceTableName
* @param monthStr
* @return
*/
String createArchiverTableIfNotExist(String sourceTableName, String monthStr);
/** 创建归档表
* @param tableName
* @param monthStr
* @param archiverTable
* @return
*/
String createArchiverTableIfNotExist(String tableName, String monthStr, ArchiverTable archiverTable);
/** 执行归档
* @param sourceTableName
* @param startTime
* @param endTime
* @return
*/
boolean doArchiver(String sourceTableName, Date startTime, Date endTime);
/**
* 扫描以表名为前缀的所有归档表
* @param sourceTableName
* @return list
*/
List<String> scanArchiver(String sourceTableName, String dbName);
/**
* 执行dump命令
* @param archiverTable
* @return
*/
String executeDump(ArchiverTable archiverTable);
/**
* drop 过期的数据库表
* @param archiverTable
* @return
*/
String dropTable(ArchiverTable archiverTable);
}

85
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/ISQLBuilder.java Normal file
View File

@@ -0,0 +1,85 @@
package top.lidee.taie.archiver.sqlbuilder;
import top.lidee.taie.archiver.config.ArchiverTable;
import java.util.Date;
/**
* SQL语句构建类接口定义
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public interface ISQLBuilder {
/** 判断表名是否存在
* @param tableName
* @return
*/
String existTable(String tableName);
/** 判断表中列名是否存在
* @param tableName
* @param fieldName
* @return
*/
String existTableField(String tableName, String fieldName);
/** 获取数据库当前时间
* @return
*/
String getNow();
/** 统计有多少条数据要归档
* @param sourceTableName
* @return
*/
String countRows(String sourceTableName);
/** 查询本次归档的任务，跨了哪几个月份
* @param tablename
* @param timefield
* @param maxDaysBeforeArchive
* @return
*/
String getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive);
/**创建归档表
* @param archiveTableName
* @param sourceTableName
* @return sql
*/
String createArchiverTableIfNotExist(String archiveTableName, String sourceTableName);
/**执行归档
* @param sourceTableName
* @param startTime
* @param endTime
* @return
*/
String doArchiver(String sourceTableName, Date startTime, Date endTime);
/**
* 扫描以表名为前缀的所有归档表
* @param sourceTableName
* @return
*/
String scanArchive(String sourceTableName, String dbName);
default String scanArchive(String sourceTableName, String dbName,String suffix){
return scanArchive(sourceTableName+suffix,dbName);
};
/**
* 执行dump命令
* @param archiverTable
* @return
*/
String executeDump(ArchiverTable archiverTable);
/**
* drop 表
* @param archiverTable
* @return
*/
String dropTable(ArchiverTable archiverTable);
}

21
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/SQLBuilderFactory.java Normal file
View File

@@ -0,0 +1,21 @@
package top.lidee.taie.archiver.sqlbuilder;
import top.lidee.taie.archiver.consant.DBType;
import java.util.Optional;
/**
* SQL构建器工厂
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class SQLBuilderFactory {
private static final SQLBuilderRegistry sqlBuilderRegistry = new SQLBuilderRegistry();
public static ISQLBuilder getSQLBuilder(DBType dbType){
return Optional.ofNullable(sqlBuilderRegistry.getSQLBuilder(dbType)).orElseThrow(()->
new RuntimeException(String.format("%s database not supported", dbType.getDb()))
);
}
}

36
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/SQLBuilderRegistry.java Normal file
View File

@@ -0,0 +1,36 @@
package top.lidee.taie.archiver.sqlbuilder;
import top.lidee.taie.archiver.consant.DBType;
import top.lidee.taie.archiver.sqlbuilder.builders.MariadbBuilder;
import top.lidee.taie.archiver.sqlbuilder.builders.MysqlBuilder;
import top.lidee.taie.archiver.sqlbuilder.builders.OracleBuilder;
import top.lidee.taie.archiver.sqlbuilder.builders.SqlserverBuilder;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* SQL构建器 注册类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class SQLBuilderRegistry {
private final Map<DBType, ISQLBuilder> sqlBuilderMap = new HashMap<DBType, ISQLBuilder>();
public SQLBuilderRegistry(){
this.sqlBuilderMap.put(DBType.MYSQL, new MysqlBuilder());
this.sqlBuilderMap.put(DBType.MARIADB, new MariadbBuilder());
this.sqlBuilderMap.put(DBType.ORACLE, new OracleBuilder());
this.sqlBuilderMap.put(DBType.SQL_SERVER, new SqlserverBuilder());
}
public ISQLBuilder getSQLBuilder(DBType dbType){
return this.sqlBuilderMap.get(dbType);
}
public Collection<ISQLBuilder> getSQLBuilders(){
return Collections.unmodifiableCollection(this.sqlBuilderMap.values());
}
}

88
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/builders/MariadbBuilder.java Normal file
View File

@@ -0,0 +1,88 @@
package top.lidee.taie.archiver.sqlbuilder.builders;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.sqlbuilder.ISQLBuilder;
import java.util.Date;
/**
* mariadb相关sql语句
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class MariadbBuilder implements ISQLBuilder {
@Override
public String existTable(String tableName) {
String sql = String.format("desc %s", tableName);
return sql;
}
@Override
public String existTableField(String tableName, String fieldName) {
String sql = String.format("desc %s", tableName);
return sql;
}
@Override
public String getNow() {
String sql="select now() as now";
return sql;
}
@Override
public String countRows(String sourceTableName) {
return null;
}
@Override
public String getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive) {
String sql = String.format("select date_format(%s,'%%Y-%%m') month from %s where %s < DATE_SUB(NOW(), interval %d day) group by date_format(%s,'%%Y-%%m')", timefield, tablename, timefield, maxDaysBeforeArchive, timefield);
return sql;
}
@Override
public String createArchiverTableIfNotExist(String archiveTableName, String sourceTableName) {
String sql = String.format("create table %s like %s", archiveTableName, sourceTableName);
return sql;
}
@Override
public String doArchiver(String sourceTableName, Date startTime, Date endTime) {
return null;
}
/**
* 扫描以表名为前缀的所有归档表
*
* @param sourceTableName
* @return
*/
@Override
public String scanArchive(String sourceTableName, String dbName) {
return null;
}
/**
* 执行dump命令
*
* @param archiverTable
* @return
*/
@Override
public String executeDump(ArchiverTable archiverTable) {
return null;
}
/**
* drop 表
*
* @param archiverTable
* @return
*/
@Override
public String dropTable(ArchiverTable archiverTable) {
return null;
}
}

133
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/builders/MysqlBuilder.java Normal file
View File

@@ -0,0 +1,133 @@
package top.lidee.taie.archiver.sqlbuilder.builders;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.sqlbuilder.ISQLBuilder;
import top.lidee.taie.archiver.utils.DateUtil;
import top.lidee.taie.archiver.utils.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Date;
import java.util.List;
/**
* mysql相关sql语句
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class MysqlBuilder implements ISQLBuilder {
private Logger log = LoggerFactory.getLogger(MysqlBuilder.class);
@Override
public String existTable(String tableName) {
String sql = String.format("desc %s", tableName);
return sql;
}
@Override
public String existTableField(String tableName, String fieldName) {
String sql = String.format("desc %s", tableName);
return sql;
}
@Override
public String getNow() {
String sql = "select now() as now";
return sql;
}
@Override
public String countRows(String sourceTableName) {
return null;
}
@Override
public String getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive) {
String sql = String.format("select date_format(%s,'%%Y-%%m') month from %s where %s < DATE_SUB(NOW(), interval %d day) group by date_format(%s,'%%Y-%%m')", timefield, tablename, timefield, maxDaysBeforeArchive, timefield);
return sql;
}
@Override
public String createArchiverTableIfNotExist(String archiveTableName, String sourceTableName) {
String sql = String.format("create table %s like %s", archiveTableName, sourceTableName);
return sql;
}
@Override
public String doArchiver(String sourceTableName, Date startTime, Date endTime) {
return null;
}
/**
* 扫描以表名为前缀的所有归档表
*
* @param sourceTableName
* @return
*/
@Override
public String scanArchive(String sourceTableName, String dbName) {
String sql = String.format(
"SELECT table_name, table_type, engine FROM information_schema.tables WHERE table_schema = '%s' and table_name like '%s'",
dbName, sourceTableName+"%");
return sql;
}
/**
* 执行dump命令
* mysqldump --set-gtid-purged=OFF --single-transaction -h10.108.26.197 -P3306 -uroot -pappuser@anji --databases lidee_auth --tables lidee_log lidee_log_202003 lidee_log_202007 > db.sql
*
* @param archiverTable
* @return
*/
@Override
public String executeDump(ArchiverTable archiverTable) {
StringBuilder builder = new StringBuilder();
String dumpFile = archiverTable.getDumpFile();
File file = new File(dumpFile + File.separator + DateUtil.now() + ".sql");
File fileParent = file.getParentFile();
if(!fileParent.exists()){
fileParent.mkdirs();
}
builder.append("mysqldump --set-gtid-purged=OFF --single-transaction -h").append(archiverTable.getDumpHost())
.append(" -P").append(archiverTable.getDumpPort())
.append(" -u").append(archiverTable.getDumpUser())
.append(" -p").append(archiverTable.getDumpPasswd())
.append(" --databases ").append(archiverTable.getDbName())
.append(" --tables ");
archiverTable.getDumpTableList().forEach(tableName -> {
builder.append(tableName).append(" ");
});
builder.append(" > ").append(file.getPath());
log.info("mysql execute dump：{}", builder);
String result = SystemUtils.exeCmd(builder.toString());
log.info("execute result：{}", result);
return result;
}
/**
* drop 表
*
* @param archiverTable
* @return
*/
@Override
public String dropTable(ArchiverTable archiverTable) {
List<String> list = archiverTable.getDumpTableList();
StringBuilder sb = new StringBuilder();
sb.append("drop table ");
for (int i = 0; i < list.size(); i++) {
if (i < list.size() - 1) {
sb.append(list.get(i));
sb.append(",");
} else {
sb.append(list.get(i));
}
}
log.info("drop sql：{}", sb);
return sb.toString();
}
}

81
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/builders/OracleBuilder.java Normal file
View File

@@ -0,0 +1,81 @@
package top.lidee.taie.archiver.sqlbuilder.builders;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.sqlbuilder.ISQLBuilder;
import java.util.Date;
/**
* oracle相关sql语句
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class OracleBuilder implements ISQLBuilder {
@Override
public String existTable(String tableName) {
return null;
}
@Override
public String existTableField(String tableName, String fieldName) {
return null;
}
@Override
public String getNow() {
return null;
}
@Override
public String countRows(String sourceTableName) {
return null;
}
@Override
public String getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive) {
return null;
}
@Override
public String createArchiverTableIfNotExist(String archiverTableName, String sourceTableName) {
return null;
}
@Override
public String doArchiver(String sourceTableName, Date startTime, Date endTime) {
return null;
}
/**
* 扫描以表名为前缀的所有归档表
*
* @param sourceTableName
* @return
*/
@Override
public String scanArchive(String sourceTableName, String dbName) {
return null;
}
/**
* 执行dump命令
*
* @param archiverTable
* @return
*/
@Override
public String executeDump(ArchiverTable archiverTable) {
return null;
}
/**
* drop 表
*
* @param archiverTable
* @return
*/
@Override
public String dropTable(ArchiverTable archiverTable) {
return null;
}
}

81
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/sqlbuilder/builders/SqlserverBuilder.java Normal file
View File

@@ -0,0 +1,81 @@
package top.lidee.taie.archiver.sqlbuilder.builders;
import top.lidee.taie.archiver.config.ArchiverTable;
import top.lidee.taie.archiver.sqlbuilder.ISQLBuilder;
import java.util.Date;
/**
* ms server相关sql语句
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class SqlserverBuilder implements ISQLBuilder {
@Override
public String existTable(String tableName) {
return null;
}
@Override
public String existTableField(String tableName, String fieldName) {
return null;
}
@Override
public String getNow() {
return null;
}
@Override
public String countRows(String sourceTableName) {
return null;
}
@Override
public String getCrossMonthList(String tablename, String timefield, Integer maxDaysBeforeArchive) {
return null;
}
@Override
public String createArchiverTableIfNotExist(String archiverTableName, String sourceTableName) {
return null;
}
@Override
public String doArchiver(String sourceTableName, Date startTime, Date endTime) {
return null;
}
/**
* 扫描以表名为前缀的所有归档表
*
* @param sourceTableName
* @return
*/
@Override
public String scanArchive(String sourceTableName, String dbName) {
return null;
}
/**
* 执行dump命令
*
* @param archiverTable
* @return
*/
@Override
public String executeDump(ArchiverTable archiverTable) {
return null;
}
/**
* drop 表
*
* @param archiverTable
* @return
*/
@Override
public String dropTable(ArchiverTable archiverTable) {
return null;
}
}

96
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/utils/DateUtil.java Normal file
View File

@@ -0,0 +1,96 @@
package top.lidee.taie.archiver.utils;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
/**
* 日期工具类
*
* @author: Devli
* @since 2021/2/3 14:16
*/
public class DateUtil {
private static String defaultDatePattern = "yyyy-MM-dd";
public static Date addMonth(String date, int increase) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date result = sdf.parse(date);
return addMonth(result, increase);
} catch (Exception e) {
return null;
}
}
public static String now(){
return format(new Date(), "yyyyMMddHHmmssSSS");
}
public static Date addMonth(Date date, int increase) {
Calendar rightNow = Calendar.getInstance();
rightNow.setTime(date);
rightNow.add(Calendar.MONTH, increase);
Date result = rightNow.getTime();
return result;
}
public static Date addDays(Date date, int increase) {
Calendar rightNow = Calendar.getInstance();
rightNow.setTime(date);
rightNow.add(Calendar.DAY_OF_YEAR, increase);
Date result = rightNow.getTime();
return result;
}
public static String format(Date date, String pattern) {
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
return date == null ? null : sdf.format(date);
}
public static String formatDate(Date date) {
return format(date, defaultDatePattern);
}
public static Date parse(String dateStr, String pattern) {
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
if (dateStr != null && !"".equals(dateStr)) {
try {
Date d = sdf.parse(dateStr);
return d;
} catch (Exception e) {
e.printStackTrace();
return null;
}
} else {
return null;
}
}
/**
* 获取几天前的时间
* @param d
* @param day
* @return
*/
public static Date getDateBefore(Date d, int day) {
Calendar now = Calendar.getInstance();
now.setTime(d);
now.set(Calendar.DATE, now.get(Calendar.DATE) - day);
return now.getTime();
}
/**
* 比较时间大小
* d1早于d2 返回true
* @param d1
* @param d2
* @return
*/
public static boolean before(Date d1, Date d2) {
return d1.getTime() <= d2.getTime();
}
}

124
lidee-common/lidee-boot-archiver/src/main/java/top/lidee/taie/archiver/utils/SystemUtils.java Normal file
View File

@@ -0,0 +1,124 @@
package top.lidee.taie.archiver.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.Properties;
/**
* Created by raodeming on 2021/4/20.
*/
public class SystemUtils {
private static final Logger log = LoggerFactory.getLogger(SystemUtils.class);
public static boolean isLinux() {
Properties prop = System.getProperties();
String os = prop.getProperty("os.name");
if (os != null && os.toLowerCase().indexOf("linux") > -1) {
return true;
} else {
return false;
}
}
public static boolean isWindows() {
Properties prop = System.getProperties();
String os = prop.getProperty("os.name");
if (os != null && os.toLowerCase().indexOf("window") > -1) {
return true;
} else {
return false;
}
}
/***
* @param sb 命令脚本
*
*/
public static String runCmd(String sb) {
boolean isLinux = SystemUtils.isLinux();
Process process = null;
String encoding = "UTF-8";
try {
if (isLinux) {
encoding = "UTF-8";
String[] cmds = new String[]{"/bin/sh", "-c", sb};
process = Runtime.getRuntime().exec(cmds);
} else {
encoding = "GBK";
String str = "cmd /c " + sb;
process = Runtime.getRuntime().exec(str);
}
try (SequenceInputStream sis = new SequenceInputStream(process.getInputStream(),
process.getErrorStream());
InputStreamReader isr = new InputStreamReader(sis, encoding);
BufferedReader br = new BufferedReader(isr);
OutputStreamWriter osw = new OutputStreamWriter(process.getOutputStream());
BufferedWriter bw = new BufferedWriter(osw)
) {
String line = null;
while (null != (line = br.readLine())) {
log.info("=============>"+line);
if (!line.contains("Warning")) {
throw new RuntimeException(line);
}
}
bw.flush();
}
int ret = process.waitFor();
log.info("---process-run-return:{}",ret);
return ret+"";
} catch (Exception ex) {
log.error("cmd-exec-error:{}", sb, ex);
return ex.getCause()==null?ex.getMessage():ex.getCause().getMessage();
} finally {
if (process != null) {
process.destroy();
log.info("cmd-exec:{},ret:{}", sb, process.exitValue());
}
}
}
public static String exeCmd(String commandStr) {
boolean isLinux = SystemUtils.isLinux();
String result = null;
Process process;
String encoding;
try {
if (isLinux) {
encoding = "UTF-8";
String[] cmd = new String[]{"/bin/sh", "-c",commandStr};
process = Runtime.getRuntime().exec(cmd);
} else {
encoding = "GBK";
String cmd = "cmd /c " + commandStr;
process = Runtime.getRuntime().exec(cmd);
}
StringBuffer sb = new StringBuffer();
try (SequenceInputStream sis = new SequenceInputStream(process.getInputStream(),
process.getErrorStream());
InputStreamReader isr = new InputStreamReader(sis, encoding);
BufferedReader br = new BufferedReader(isr);
OutputStreamWriter osw = new OutputStreamWriter(process.getOutputStream());
BufferedWriter bw = new BufferedWriter(osw)
) {
String line;
while (null != (line = br.readLine())) {
sb.append(line).append("</br>");
}
bw.flush();
}
result = sb.toString();
} catch (Exception e) {
log.error("exec error", e);
}
return result;
}
}

1
lidee-common/lidee-boot-archiver/src/main/resources/META-INF/spring.factories Normal file
View File

@@ -0,0 +1 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=top.lidee.taie.archiver.config.AutoConfiguration

12
lidee-common/lidee-boot-archiver/src/main/resources/application.properties Normal file
View File

@@ -0,0 +1,12 @@
spring.lidee.subscribes.archiver.enabled=false
#归档触发的定时器，默认每月2号，凌晨3:0:0执行归档
spring.lidee.subscribes.archiver.archive-scheduled-cron=0 0 3 2 */1 ?
#归档多久之前的数据，将n天前的数据，移动到归档表，tables配置项中策略优先加载
spring.lidee.subscribes.archiver.max-days-before-archive=30
#删除多久之前的历史数据，归档表中，已归档的数据保留期限，超过期限的数据将删除。tables配置项中策略优先加载
spring.lidee.subscribes.archiver.max-days-before-delete=720
spring.lidee.subscribes.archiver.tables[0].tablename=
spring.lidee.subscribes.archiver.tables[0].timefield=
spring.lidee.subscribes.archiver.tables[0].max-days-before-archive=
spring.lidee.subscribes.archiver.tables[0].max-days-before-delete=

14
lidee-common/lidee-boot-archiver/src/main/resources/bootstrap.yml Normal file
View File

@@ -0,0 +1,14 @@
spring:
lidee:
subscribes:
archiver:
enabled: false
archive-scheduled-cron: 0 0 3 2 */1 ? #归档触发的定时器默认每月2号，凌晨3:0:0执行归档
max-days-before-archive: 30 #归档多久之前的数据，将n天前的数据，移动到归档表，tables配置项中策略优先加载
max-days-before-delete: 720 #删除多久之前的历史数据，归档表中，已归档的数据保留期限，超过期限的数据将删除。tables配置项中策略优先加载
execute-ip: '' #限制执行的服务ip
tables:
- tablename:
timefield:
max-days-before-archive:
max-days-before-delete: