Merge branch 'develop' into dev/performance_analysis

.devcontainer/devcontainer.json

@@ -0,0 +1,9 @@
+{
+  "image": "mcr.microsoft.com/devcontainers/universal:2",
+  "features": {
+    "ghcr.io/devcontainers/features/java:1": {
+      "version": "8",
+      "installGradle": true
+    }
+  }
+}

femas-adaptor/femas-adaptor-opensource-admin/src/main/java/com/tencent/tsf/femas/adaptor/paas/config/FemasConfigGrpcClientManager.java

@@ -27,11 +27,15 @@ public class FemasConfigGrpcClientManager implements ServerConnectorManager {
 
     public FemasConfigGrpcClientManager() {
         String host = GrpcHepler.getPaasServerHost();
-        int port = GrpcHepler.getPaasGrpcPort();
-        channel = NettyChannelBuilder
-                .forAddress(host, port)
-                .usePlaintext()
-                .build();
+        if(StringUtils.isNotEmpty(host)){
+            int port = GrpcHepler.getPaasGrpcPort();
+            channel = NettyChannelBuilder
+                    .forAddress(host, port)
+                    .usePlaintext()
+                    .build();
+        }else {
+            log.warn("skipping initialization of grpc channel, because doesn't found the femas address configuration");
+        }
     }
 
     @Override

femas-adaptor/femas-adaptor-opensource-admin/src/main/java/com/tencent/tsf/femas/adaptor/paas/config/FemasConfigHttpClientManager.java

@@ -6,8 +6,10 @@
 import com.tencent.tsf.femas.common.entity.Service;
 import com.tencent.tsf.femas.common.exception.FemasRuntimeException;
 import com.tencent.tsf.femas.common.httpclient.ApacheHttpClientHolder;
+import com.tencent.tsf.femas.common.httpclient.HttpLongPollingConnectorManager;
 import com.tencent.tsf.femas.common.httpclient.client.AbstractHttpClient;
 import com.tencent.tsf.femas.common.httpclient.factory.ApacheDefaultHttpClientFactory;
+import com.tencent.tsf.femas.common.httpclient.factory.ApacheLongPollingHttpClientFactory;
 import com.tencent.tsf.femas.common.httpclient.factory.HttpClientFactory;
 import com.tencent.tsf.femas.common.util.CollectionUtil;
 import com.tencent.tsf.femas.common.util.HttpHeaderKeys;
@@ -38,19 +40,24 @@
 import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
 
-public class FemasConfigHttpClientManager implements ServerConnectorManager {
+public class FemasConfigHttpClientManager implements ServerConnectorManager, HttpLongPollingConnectorManager {
 
     private static final Logger log = LoggerFactory.getLogger(FemasConfigHttpClientManager.class);
 
     private static final String webContext = "/atom";
     private static final String fetchKeyUrl = "/v1/sdk/fetchData";
+    private static final String longPollingListenerKeyUrl = "/v1/sdk/listener";
     private static final String reportCircuitEvent = "/v1/sdk/reportServiceEvent";
     private static final String reportApis = "/v1/sdk/reportServiceApi";
     private static final String intiNamespace = "/v1/sdk/initNamespace";
     private static final int DEFAULT_READ_TIME_OUT_MILLIS = Integer
             .getInteger("femas.paas.config.client.readTimeOut", 50000);
     private static final int DEFAULT_CON_TIME_OUT_MILLIS = Integer
             .getInteger("femas.paas.config.client.conTimeOut", 3000);
+    private static final int DEFAULT_LONG_POLLING_READ_TIME_OUT_MILLIS = Integer
+            .getInteger("femas.paas.config.client.longPolling.readTimeOut", 900000);
+    private static final int DEFAULT_LONG_POLLING_CON_TIME_OUT_MILLIS = Integer
+            .getInteger("femas.paas.config.client.longPolling.conTimeOut", 3000);
     private static Context commonContext = ContextFactory.getContextInstance();
     private static volatile FemasConfigHttpClientManager singleton = null;
     private volatile Context context = ContextFactory.getContextInstance();
@@ -75,15 +82,19 @@ public void destroy() {
      */
     private String paasServerDomain;
     private String keyListenerUrl;
+    private String longPollingKeyListenerUrl;
     private String reportCircuitEventUrl;
     private String reportApisUrl;
     private String initNamespaceUrl;
     private AbstractHttpClient httpClient;
+    private AbstractHttpClient httpLongPollingClient;
 
     public FemasConfigHttpClientManager() {
         Map<String, String> configMap = commonContext.getRegistryConfigMap();
         HttpClientFactory httpClientFactory = new ApacheDefaultHttpClientFactory(DEFAULT_READ_TIME_OUT_MILLIS,
                 DEFAULT_CON_TIME_OUT_MILLIS);
+        HttpClientFactory httpLongPollingFactory = new ApacheLongPollingHttpClientFactory(DEFAULT_LONG_POLLING_CON_TIME_OUT_MILLIS,
+                DEFAULT_LONG_POLLING_READ_TIME_OUT_MILLIS);
 
         String paasServerDomainByConfig = null;
         if (configMap != null) {
@@ -94,10 +105,12 @@ public FemasConfigHttpClientManager() {
         }
         this.paasServerDomain = paasServerDomainByConfig;
         this.keyListenerUrl = paasServerDomain.concat(webContext).concat(fetchKeyUrl);
+        this.longPollingKeyListenerUrl = paasServerDomain.concat(webContext).concat(longPollingListenerKeyUrl);
         this.reportCircuitEventUrl = paasServerDomain.concat(webContext).concat(reportCircuitEvent);
         this.reportApisUrl = paasServerDomain.concat(webContext).concat(reportApis);
         this.initNamespaceUrl = paasServerDomain.concat(webContext).concat(intiNamespace);
         this.httpClient = ApacheHttpClientHolder.getHttpClient(httpClientFactory);
+        this.httpLongPollingClient = ApacheHttpClientHolder.getHttpClient(httpLongPollingFactory);
     }
 
     @Override
@@ -232,4 +245,21 @@ public void initNamespace(String registryAddress, String namespaceId) {
         }
     }
 
+    @Override
+    public HttpResult<String> fetchLongPollingKvValue(String key, String namespaceId) {
+        final Map<String, Object> params = new HashMap<>(3);
+        params.put("namespaceId", namespaceId);
+        params.put("key", key);
+        if (context.isEmptyPaasServer()) {
+            log.debug("fetchLongPollingKvValue failed , could not find the paas address profile");
+            return null;
+        }
+        HttpResult<String> httpResult = null;
+        try {
+            httpResult = httpLongPollingClient.get(longPollingKeyListenerUrl, builderHeader(), params);
+        } catch (Exception e) {
+            log.error("config http manager fetchLongPollingKvValue failed", e);
+        }
+        return httpResult;
+    }
 }

femas-adaptor/femas-adaptor-opensource-admin/src/main/java/com/tencent/tsf/femas/adaptor/paas/governance/lane/FemasLaneFilter.java

@@ -17,7 +17,6 @@
 import com.tencent.tsf.femas.governance.lane.LaneFilter;
 import com.tencent.tsf.femas.plugin.context.ConfigContext;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -143,10 +142,11 @@ private static List<ServiceInstance> chooseColorlessInstances(Service service,
             LOGGER.debug("[FEMAS LANE] Choose Colorless instances. lane take effect, filter color instance list = {}",
                     colorInstances);
         }
-        if (CollectionUtils.isEmpty(instances)) {
+        if (CollectionUtil.isEmpty(instances)) {
             return serviceInstances;
         }
         return instances;
+
     }
 
     public static synchronized void addLaneInfo(LaneInfo laneInfo) {
@@ -172,7 +172,6 @@ public static synchronized void addLaneInfo(LaneInfo laneInfo) {
             NAMESPACE_LANE_INFO_MAP.get(serviceInfo.getNamespaceId()).add(laneInfo.getLaneId());
             String serviceGroup = getLaneConfiguredVersionKey(serviceInfo.getServiceName(), serviceInfo.getVersion());
             if (!GROUP_LANE_INFO_COLOR_MAP.containsKey(serviceGroup)) {
-//                GROUP_LANE_INFO_COLORLESS_MAP.putIfAbsent(laneGroup.getGroupId(), new HashSet<>());
                 GROUP_LANE_INFO_COLOR_MAP.putIfAbsent(serviceGroup, new HashSet<>());
             }
 //            /**
@@ -191,13 +190,6 @@ public static synchronized void addLaneInfo(LaneInfo laneInfo) {
              * 所以这里删除所有为false的Service
              */
             EFFECTIVE_SERVICE_MAP.clear();
-//            Iterator<Map.Entry<Service, Boolean>> iterator = EFFECTIVE_SERVICE_MAP.entrySet().iterator();
-//            while (iterator.hasNext()) {
-//                Map.Entry<Service, Boolean> entry = iterator.next();
-//                if (entry.getValue() == false) {
-//                    iterator.remove();
-//                }
-//            }
         }
     }
 
@@ -220,24 +212,15 @@ public static synchronized void removeLaneInfo(LaneInfo laneInfo) {
             String serviceGroup = getLaneConfiguredVersionKey(serviceInfo.getServiceName(), serviceInfo.getVersion());
 
             if (!GROUP_LANE_INFO_COLOR_MAP.containsKey(serviceGroup)) {
-//                GROUP_LANE_INFO_COLORLESS_MAP.putIfAbsent(laneGroup.getGroupId(), new HashSet<>());
                 GROUP_LANE_INFO_COLOR_MAP.putIfAbsent(serviceGroup, new HashSet<>());
             }
-//            GROUP_LANE_INFO_COLORLESS_MAP.get(laneGroup.getGroupId()).remove(laneInfo.getLaneId());
             GROUP_LANE_INFO_COLOR_MAP.get(serviceGroup).remove(laneInfo.getLaneId());
 
             /**
              * 删除泳道可能会导致原来配置过泳道的服务变为没有配置过
              * 所以这里删除所有为true的Service
              */
             EFFECTIVE_SERVICE_MAP.clear();
-//            Iterator<Map.Entry<Service, Boolean>> iterator = EFFECTIVE_SERVICE_MAP.entrySet().iterator();
-//            while (iterator.hasNext()) {
-//                Map.Entry<Service, Boolean> entry = iterator.next();
-//                if (entry.getValue() == true) {
-//                    iterator.remove();
-//                }
-//            }
         }
     }
 
@@ -267,14 +250,11 @@ private static synchronized void refreshEffectiveLaneRule() {
     private static synchronized void resortLaneRule() {
         List<LaneRule> laneRules = new ArrayList<>(EFFECTIVE_LANE_RULES_SET);
         EFFECTIVE_LANE_RULES = laneRules;
-        Collections.sort(EFFECTIVE_LANE_RULES, new Comparator<LaneRule>() {
-            @Override
-            public int compare(LaneRule r1, LaneRule r2) {
-                if (r1.getPriority().equals(r2.getPriority())) {
-                    return Long.compare(r2.getCreateTime().getTime(), r1.getCreateTime().getTime());
-                } else {
-                    return r1.getPriority().compareTo(r2.getPriority());
-                }
+        Collections.sort(EFFECTIVE_LANE_RULES, (r1, r2) -> {
+            if (r1.getPriority().equals(r2.getPriority())) {
+                return Long.compare(r2.getCreateTime().getTime(), r1.getCreateTime().getTime());
+            } else {
+                return r1.getPriority().compareTo(r2.getPriority());
             }
         });
     }
@@ -377,29 +357,18 @@ public void preProcessLaneId() {
     }
 
     private String getLaneIdByPercentage(Map<String, Integer> laneMap) {
-        Map<Integer, String> reverseMap = new HashMap<>();
-        List<Integer> integers = new ArrayList<>();
-        laneMap.entrySet().stream().forEach(e -> {
-            reverseMap.put(e.getValue(), e.getKey());
-            integers.add(e.getValue());
-        });
-        Collections.sort(integers, new Comparator<Integer>() {
-            @Override
-            public int compare(Integer r1, Integer r2) {
-                return r1 - r2;
-            }
-        });
-        Random random = new Random();
-        Integer index = random.nextInt(100);
+        TreeMap<Integer, String> weightMap = new TreeMap<>();
         int cur = 0;
-        int res = 0;
-        for (int a = 0; a < integers.size(); a++) {
-            cur += integers.get(a);
-            if (index < cur) {
-                res = integers.get(a);
-            }
+        for (Map.Entry<String, Integer> entry : laneMap.entrySet()) {
+            cur += entry.getValue();
+            weightMap.put(cur, entry.getKey());
         }
-        return reverseMap.get(res);
+
+        Random random = new Random();
+        Integer index = random.nextInt(100);
+
+        SortedMap<Integer, String> tailMap = weightMap.tailMap(index, false);
+        return  weightMap.get(tailMap.firstKey());
     }
 
 

femas-adaptor/femas-adaptor-opensource-admin/src/main/java/com/tencent/tsf/femas/adaptor/paas/governance/lane/LaneInfo.java

@@ -1,6 +1,7 @@
 package com.tencent.tsf.femas.adaptor.paas.governance.lane;
 
 import java.util.List;
+import java.util.Objects;
 
 /**
  * 泳道
@@ -85,4 +86,21 @@ public List<ServiceInfo> getLaneServiceList() {
     public void setLaneServiceList(List<ServiceInfo> laneServiceList) {
         this.laneServiceList = laneServiceList;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        LaneInfo laneInfo = (LaneInfo) o;
+        return Objects.equals(laneId, laneInfo.laneId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(laneId);
+    }
 }

femas-adaptor/femas-adaptor-opensource-admin/src/main/java/com/tencent/tsf/femas/adaptor/paas/governance/lane/ServiceInfo.java

@@ -1,5 +1,7 @@
 package com.tencent.tsf.femas.adaptor.paas.governance.lane;
 
+import java.util.Optional;
+
 /**
  * @Author: cody
  * @Date: 2022/7/26
@@ -50,7 +52,8 @@ public void setNamespaceName(String namespaceName) {
     }
 
     public Boolean getEntrance() {
-        return entrance;
+        return Optional.ofNullable(entrance)
+                .orElse(false);
     }
 
     public void setEntrance(Boolean entrance) {

femas-admin/src/main/java/com/tencent/tsf/femas/constant/AdminConstants.java

@@ -49,6 +49,9 @@ public class AdminConstants {
 
     public static final String FEMAS_K8S_SELECT_LABEL_KEY = "femas-service-app=";
 
+    public static final String LOCALHOST_STRING = "localhost";
+    public static final String LOCALHOST_IP = "127.0.0.1";
+
     public static class StorageKeyPrefix {
 
         public static final String REGISTRY_CONFIG_PREFIX = CONTEXT_PATH_PREFIX.concat("registry/config/");

femas-admin/src/main/java/com/tencent/tsf/femas/endpoint/adaptor/ControllerExecutorTemplate.java

@@ -99,7 +99,7 @@ private void saveRecord(boolean status, String endPointPath, String method, Obje
             return;
         }
         Record record = new Record();
-        record.setTime(new Date().getTime());
+        record.setTime(System.currentTimeMillis());
         record.setStatus(status);
         record.setModule(module);
         record.setUser("admin");

femas-admin/src/main/java/com/tencent/tsf/femas/endpoint/configlisten/LongPollingEndpoint.java

@@ -3,15 +3,10 @@
 import com.tencent.tsf.femas.constant.IgnorePrefix;
 import com.tencent.tsf.femas.endpoint.adaptor.AbstractBaseEndpoint;
 import com.tencent.tsf.femas.entity.registry.ServiceApiRequest;
-import com.tencent.tsf.femas.enums.ServiceInvokeEnum;
+import com.tencent.tsf.femas.service.http.HttpLongPollingDataUpdateService;
 import com.tencent.tsf.femas.service.namespace.NamespaceMangerService;
 import com.tencent.tsf.femas.service.registry.ServiceManagerService;
 import com.tencent.tsf.femas.service.rule.ConvertService;
-import com.tencent.tsf.femas.storage.StorageResult;
-import com.tencent.tsf.femas.storage.rocksdb.StringRawKVStoreManager;
-
-import java.util.HashMap;
-import java.util.List;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -20,6 +15,8 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import javax.servlet.http.HttpServletRequest;
+
 
 /**
  * @Author leoziltong、cody
@@ -41,19 +38,29 @@ public class LongPollingEndpoint extends AbstractBaseEndpoint {
 
     private final NamespaceMangerService namespaceMangerService;
 
+    private final HttpLongPollingDataUpdateService httpLongPollingDataUpdateService;
+
     public LongPollingEndpoint(ConvertService convertService,
-                               ServiceManagerService serviceManagerService,NamespaceMangerService namespaceMangerService) {
+                               ServiceManagerService serviceManagerService,
+                               NamespaceMangerService namespaceMangerService,
+                               HttpLongPollingDataUpdateService httpLongPollingDataUpdateService) {
 //        this.kvStoreManager = kvStoreManager;
         this.convertService = convertService;
         this.serviceManagerService = serviceManagerService;
         this.namespaceMangerService = namespaceMangerService;
+        this.httpLongPollingDataUpdateService = httpLongPollingDataUpdateService;
     }
 
     @GetMapping("fetchData")
     public String fetchBreakerRule(String key) {
         return convertService.convert(key);
     }
 
+    @GetMapping("listener")
+    public void longPollingListener(final HttpServletRequest request, String key) {
+        httpLongPollingDataUpdateService.doLongPolling(convertService.getKeyType(key), request);
+    }
+
 //    @GetMapping("showData")
 //    public HashMap<String, String> get() {
 //        StorageResult<List<String>> result = kvStoreManager.scanPrefix("");