Skip to content

Commit

Permalink
feat: 异步查询【分页】
Browse files Browse the repository at this point in the history
  • Loading branch information
@KouShenhai
KouShenhai committed Jul 5, 2024
1 parent 9ebdc94 commit 1ef0a04
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 30 deletions.
2 changes: 1 addition & 1 deletion README.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ image::doc/image/老寇IoT云平台架构图-阿里巴巴.png[架构图,align=ce
|Seata |2.0.0
|Sentinel |1.8.8
|Redis |7.2.5
|Elasticsearch |8.14.1
|Elasticsearch |8.14.2
|RocketMQ |5.2.0
|Netty |4.1.111.Final
|Kafka |3.7.1
Expand Down
52 changes: 52 additions & 0 deletions ...ybatis-plus/src/main/java/org/laokou/common/mybatisplus/config/AsyncCountInterceptor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Copyright (c) 2022-2024 KCloud-Platform-IoT Author or Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.laokou.common.mybatisplus.config;

import org.apache.ibatis.cache.CacheKey;
import org.apache.ibatis.executor.Executor;
import org.apache.ibatis.mapping.BoundSql;
import org.apache.ibatis.mapping.MappedStatement;
import org.apache.ibatis.plugin.*;
import org.apache.ibatis.session.ResultHandler;
import org.apache.ibatis.session.RowBounds;

/**
* @author gitkakafu
* @author laokou
*/
@Intercepts(value = {
@Signature(type = Executor.class, method = "query",
args = { MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class, CacheKey.class,
BoundSql.class }),
@Signature(type = Executor.class, method = "query",
args = { MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class }) })
public class AsyncCountInterceptor implements Interceptor {

@Override
public Object intercept(Invocation invocation) throws Throwable {
Object obj = invocation.proceed();
try {
AsyncPaginationInnerInterceptor.get().join();
}
finally {
AsyncPaginationInnerInterceptor.remove();
}
return obj;
}

}
57 changes: 43 additions & 14 deletions ...nfig/AysncPaginationInnerInterceptor.java → ...nfig/AsyncPaginationInnerInterceptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

package org.laokou.common.mybatisplus.config;

import com.alibaba.ttl.TransmittableThreadLocal;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.metadata.OrderItem;
Expand Down Expand Up @@ -70,6 +71,7 @@
import javax.sql.DataSource;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

Expand All @@ -79,13 +81,14 @@
* 默认对 left join 进行优化,虽然能优化count,但是加上分页的话如果1对多本身结果条数就是不正确的
* @author hubin
* @author gitkakafu
* @author laokou
* @since 3.4.0
*/
@Data
@Slf4j
@NoArgsConstructor
@SuppressWarnings({ "rawtypes" })
public class AysncPaginationInnerInterceptor implements InnerInterceptor {
public class AsyncPaginationInnerInterceptor implements InnerInterceptor {

/**
* 获取jsqlparser中count的SelectItem.
Expand All @@ -95,6 +98,16 @@ public class AysncPaginationInnerInterceptor implements InnerInterceptor {

protected static final Map<String, MappedStatement> countMsCache = new ConcurrentHashMap<>();

private static final ThreadLocal<CompletableFuture<Void>> COUNT_LOCAL = new TransmittableThreadLocal<>();

public static CompletableFuture<Void> get() {
return COUNT_LOCAL.get();
}

public static void remove() {
COUNT_LOCAL.remove();
}

/**
* 溢出总页数后是否进行处理.
*/
Expand Down Expand Up @@ -128,22 +141,28 @@ public class AysncPaginationInnerInterceptor implements InnerInterceptor {

private DataSource dataSource;

public AysncPaginationInnerInterceptor(DbType dbType, DataSource dataSource) {
private java.util.concurrent.Executor workStealingPoolExecutor;

public AsyncPaginationInnerInterceptor(DbType dbType, DataSource dataSource,
java.util.concurrent.Executor workStealingPoolExecutor) {
this.dbType = dbType;
this.dataSource = dataSource;
this.workStealingPoolExecutor = workStealingPoolExecutor;
}

public AysncPaginationInnerInterceptor(IDialect dialect, DataSource dataSource) {
public AsyncPaginationInnerInterceptor(IDialect dialect, DataSource dataSource,
java.util.concurrent.Executor workStealingPoolExecutor) {
this.dialect = dialect;
this.dataSource = dataSource;
this.workStealingPoolExecutor = workStealingPoolExecutor;
}

/**
* 这里进行count,如果count为0这返回false(就是不再执行sql了).
*/
@Override
public boolean willDoQuery(Executor executor, MappedStatement ms, Object parameter, RowBounds rowBounds,
ResultHandler resultHandler, BoundSql boundSql) throws SQLException {
ResultHandler resultHandler, BoundSql boundSql) {
IPage<?> page = ParameterUtils.findPage(parameter).orElse(null);
if (page == null || page.getSize() < 0 || !page.searchCount() || resultHandler != Executor.NO_RESULT_HANDLER) {
return true;
Expand All @@ -166,17 +185,27 @@ public boolean willDoQuery(Executor executor, MappedStatement ms, Object paramet
SimpleExecutor simpleExecutor = new SimpleExecutor(ms.getConfiguration(), springManagedTransaction);
CacheKey cacheKey = simpleExecutor.createCacheKey(countMs, parameter, rowBounds, countSql);

List<Object> result = executor.query(countMs, parameter, rowBounds, resultHandler, cacheKey, countSql);
long total = 0;
if (CollectionUtils.isNotEmpty(result)) {
// 个别数据库 count 没数据不会返回 0
Object o = result.getFirst();
if (o != null) {
total = Long.parseLong(o.toString());
final MappedStatement mappedStatement = countMs;
CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
try {
List<Object> result = executor.query(mappedStatement, parameter, rowBounds, resultHandler, cacheKey,
countSql);
long total = 0;
if (CollectionUtils.isNotEmpty(result)) {
// 个别数据库 count 没数据不会返回 0
Object o = result.getFirst();
if (o != null) {
total = Long.parseLong(o.toString());
}
}
page.setTotal(total);
}
}
page.setTotal(total);
return continuePage(page);
catch (Exception e) {
log.error("查询失败，错误信息：{}", e.getMessage(), e);
}
}, workStealingPoolExecutor);
COUNT_LOCAL.set(future);
return true;
}

@Override
Expand Down
46 changes: 31 additions & 15 deletions ...ybatis-plus/src/main/java/org/laokou/common/mybatisplus/config/MybatisPlusAutoConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.laokou.common.mybatisplus.config;

import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.autoconfigure.ConfigurationCustomizer;
import com.baomidou.mybatisplus.core.incrementer.DefaultIdentifierGenerator;
import com.baomidou.mybatisplus.core.incrementer.IdentifierGenerator;
Expand All @@ -39,6 +40,7 @@
import javax.sql.DataSource;
import java.net.InetAddress;
import java.util.Properties;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;

/**
Expand All @@ -59,21 +61,30 @@ public class MybatisPlusAutoConfig {

@Bean
public ConfigurationCustomizer slowSqlConfigurationCustomizer(SpringContextUtil springContextUtil) {
// 慢SQL
SqlMonitorInterceptor sqlMonitorInterceptor = new SqlMonitorInterceptor();
sqlMonitorInterceptor.setProperties(properties(springContextUtil));

return configuration -> configuration.addInterceptor(sqlMonitorInterceptor);
return configuration -> {
// 异步查询count
configuration.addInterceptor(new AsyncCountInterceptor());
// 慢SQL
SqlMonitorInterceptor sqlMonitorInterceptor = new SqlMonitorInterceptor();
sqlMonitorInterceptor.setProperties(properties(springContextUtil));
configuration.addInterceptor(sqlMonitorInterceptor);
};
}

// @formatter:off
/**
* 注意: 使用多个功能需要注意顺序关系,建议使用如下顺序 - 多租户,动态表名 - 分页,乐观锁 - sql 性能规范,防止全表更新与删除 总结: 对 sql
* 进行单次改造的优先放入,不对 sql 进行改造的最后放入.
* 注意: 使用多个功能需要注意顺序关系,建议使用如下顺序
* - 多租户，动态表名
* - 分页，乐观锁
* - sql性能规范，防止全表更新与删除
* 总结：对 sql进行单次改造的优先放入,不对 sql 进行改造的最后放入.
* @param mybatisPlusExtProperties mybatis配置
*/
// @formatter:on
@Bean
@ConditionalOnMissingBean(MybatisPlusInterceptor.class)
public MybatisPlusInterceptor mybatisPlusInterceptor(MybatisPlusExtProperties mybatisPlusExtProperties) {
public MybatisPlusInterceptor mybatisPlusInterceptor(MybatisPlusExtProperties mybatisPlusExtProperties,
DataSource dataSource, Executor workStealingPoolExecutor) {
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
// 数据权限插件
interceptor.addInnerInterceptor(new DataFilterInterceptor());
Expand All @@ -87,7 +98,7 @@ public MybatisPlusInterceptor mybatisPlusInterceptor(MybatisPlusExtProperties my
dynamicTableNameInnerInterceptor.setTableNameHandler(new DynamicTableNameHandler());
interceptor.addInnerInterceptor(dynamicTableNameInnerInterceptor);
// 分页插件
interceptor.addInnerInterceptor(paginationInnerInterceptor());
interceptor.addInnerInterceptor(asyncPaginationInnerInterceptor(dataSource, workStealingPoolExecutor));
// 乐观锁插件
interceptor.addInnerInterceptor(new OptimisticLockerInnerInterceptor());
// 防止全表更新与删除插件
Expand Down Expand Up @@ -125,15 +136,20 @@ public TransactionTemplate transactionTemplate(PlatformTransactionManager transa
}

/**
* 解除每页500条限制.
* 异步分页. 解除每页500条限制.
*/
private PaginationInnerInterceptor paginationInnerInterceptor() {
PaginationInnerInterceptor paginationInnerInterceptor = new PaginationInnerInterceptor();
private AsyncPaginationInnerInterceptor asyncPaginationInnerInterceptor(DataSource dataSource,
Executor workStealingPoolExecutor) {
// 使用postgresql，如果使用其他数据库，需要修改DbType
// 使用postgresql，如果使用其他数据库，需要修改DbType
// 使用postgresql，如果使用其他数据库，需要修改DbType
AsyncPaginationInnerInterceptor asyncPaginationInnerInterceptor = new AsyncPaginationInnerInterceptor(
DbType.POSTGRE_SQL, dataSource, workStealingPoolExecutor);
// -1表示不受限制
paginationInnerInterceptor.setMaxLimit(-1L);
asyncPaginationInnerInterceptor.setMaxLimit(-1L);
// 溢出总页数后是进行处理，查看源码就知道是干啥的
paginationInnerInterceptor.setOverflow(true);
return paginationInnerInterceptor;
asyncPaginationInnerInterceptor.setOverflow(true);
return asyncPaginationInnerInterceptor;
}

private Properties properties(SpringContextUtil springContextUtil) {
Expand Down

0 comments on commit 1ef0a04

Please sign in to comment.