diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..cb96f044b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +## In what area(s)? + + + +> /area runtime +> /area operator +> /area placement +> /area docs +> /area test-and-release + +## Describe the feature + diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 000000000..73ca425dc --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,28 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build with Maven + run: mvn clean install -DskipTests -B -V + && sh ./tools/check_format.sh + && mvn clean test + - name: Codecov + uses: codecov/codecov-action@v1 diff --git a/.travis.yml b/.travis.yml index cb53cacfe..e8363e468 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: java sudo: false +dist: trusty + jdk: - oraclejdk8 @@ -12,4 +14,4 @@ script: - sh ./tools/check_format.sh after_success: -- bash <(curl -s https://codecov.io/bash) \ No newline at end of file +- bash <(curl -s https://codecov.io/bash) diff --git a/README.md b/README.md index c37c604a2..e589ab84e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # SOFARegistry -[![Build Status](https://travis-ci.com/alipay/sofa-registry.svg?branch=master)](https://travis-ci.com/alipay/sofa-registry) +[![Java CI with Maven](https://github.com/sofastack/sofa-registry/actions/workflows/maven.yml/badge.svg)](https://github.com/sofastack/sofa-registry/actions/workflows/maven.yml) ![license](https://img.shields.io/badge/license-Apache--2.0-green.svg) -[![Coverage Status](https://codecov.io/gh/alipay/sofa-registry/branch/master/graph/badge.svg)](https://codecov.io/gh/alipay/sofa-registry) -![maven](https://img.shields.io/github/release/alipay/sofa-registry.svg) +[![Coverage Status](https://codecov.io/gh/alipay/sofa-registry/branch/master/graph/badge.svg)](https://codecov.io/gh/sofastack/sofa-registry) +![maven](https://img.shields.io/github/release/sofastack/sofa-registry.svg) SOFARegistry 是蚂蚁金服开源的一个生产级、高时效、高可用的服务注册中心。SOFARegistry 最早源自于淘宝的 ConfigServer，十年来，随着蚂蚁金服的业务发展，注册中心架构已经演进至第五代。目前 SOFARegistry 不仅全面服务于蚂蚁金服的自有业务，还随着蚂蚁金融科技服务众多合作伙伴，同时也兼容开源生态。SOFARegistry 采用 AP 架构，支持秒级时效性推送，同时采用分层架构支持无限水平扩展。 -## 功能特性 +## 功能特性 - 支持服务发布与服务订阅 - 支持服务变更时的主动推送 diff --git a/client/all/pom.xml b/client/all/pom.xml index 9ab6a12dd..e39f2d502 100644 --- a/client/all/pom.xml +++ b/client/all/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-client-all - 5.2.0-SNAPSHOT + 5.4.5 ${project.groupId}:${project.artifactId} http://github.com/alipay/sofa-registry @@ -56,7 +56,7 @@ 3.3.6 1.5.2 1.0.12 - 4.1.25.Final + 4.1.42.Final 1.7.21 ../../ diff --git a/client/api/pom.xml b/client/api/pom.xml index b4f43b84b..996a538c0 100644 --- a/client/api/pom.xml +++ b/client/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/client/impl/pom.xml b/client/impl/pom.xml index e9a54b4c5..769c95d4a 100644 --- a/client/impl/pom.xml +++ b/client/impl/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/DefaultRegistryClientTest.java b/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/DefaultRegistryClientTest.java index 183ebb4ec..aba3677dc 100644 --- a/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/DefaultRegistryClientTest.java +++ b/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/DefaultRegistryClientTest.java @@ -16,6 +16,17 @@ */ package com.alipay.sofa.registry.client.provider; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.alipay.sofa.registry.client.api.ConfigDataObserver; import com.alipay.sofa.registry.client.api.Configurator; import com.alipay.sofa.registry.client.api.Publisher; @@ -31,16 +42,6 @@ import com.alipay.sofa.registry.core.model.PublisherRegister; import com.alipay.sofa.registry.core.model.ScopeEnum; import com.alipay.sofa.registry.core.model.SubscriberRegister; -import org.junit.After; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; /** * The type Default registry client test. @@ -89,7 +90,7 @@ public void registerPublisher() throws InterruptedException { defaultPublisher.republish("republish test"); - Thread.sleep(500L); + Thread.sleep(2000L); // register success when republish assertTrue(defaultPublisher.isRegistered()); } @@ -111,7 +112,7 @@ public void handleData(String dataId, UserData data) { assertNotNull(subscriber); assertEquals(dataId, subscriber.getDataId()); - Thread.sleep(500L); + Thread.sleep(2000L); RegisterCache registerCache = registryClient.getRegisterCache(); @@ -142,7 +143,7 @@ public void handleData(String dataId, ConfigData configData) { assertNotNull(configurator); assertEquals(dataId, configurator.getDataId()); - Thread.sleep(500L); + Thread.sleep(2000L); RegisterCache registerCache = registryClient.getRegisterCache(); @@ -164,7 +165,7 @@ public void unregisterSinglePublisherTest() throws InterruptedException { int unregisterCount = registryClient.unregister(dataId, null, RegistryType.PUBLISHER); assertEquals(1, unregisterCount); - Thread.sleep(500L); + Thread.sleep(2000L); Publisher temp = registryClient.getRegisterCache().getPublisherByRegistId( publisher.getRegistId()); @@ -185,7 +186,7 @@ public void unregisterSingleSubscriberTest() throws InterruptedException { int unregisterCount = registryClient.unregister(dataId, null, RegistryType.SUBSCRIBER); assertEquals(1, unregisterCount); - Thread.sleep(500L); + Thread.sleep(2000L); Subscriber temp = registryClient.getRegisterCache().getSubscriberByRegistId( subscriber.getRegistId()); @@ -206,7 +207,7 @@ public void unregisterSingleConfiguratorTest() throws InterruptedException { int unregisterCount = registryClient.unregister(dataId, null, RegistryType.CONFIGURATOR); assertEquals(1, unregisterCount); - Thread.sleep(500L); + Thread.sleep(2000L); Subscriber temp = registryClient.getRegisterCache().getSubscriberByRegistId( configurator.getRegistId()); @@ -236,13 +237,13 @@ public void unregisterMultiTest() throws InterruptedException { Subscriber subscriber2 = registryClient.register(subscriberRegistration2); - Thread.sleep(500L); + Thread.sleep(2000L); // 2. unregister publisher int unregisterCount = registryClient.unregister(dataId, null, RegistryType.PUBLISHER); assertEquals(2, unregisterCount); - Thread.sleep(500L); + Thread.sleep(2000L); // 3. check publisher register cache RegisterCache registerCache = registryClient.getRegisterCache(); @@ -256,7 +257,7 @@ public void unregisterMultiTest() throws InterruptedException { unregisterCount = registryClient.unregister(dataId, null, RegistryType.SUBSCRIBER); assertEquals(2, unregisterCount); - Thread.sleep(500L); + Thread.sleep(2000L); // 5. check subscriber register cache Subscriber tempSub = registerCache.getSubscriberByRegistId(subscriber1.getRegistId()); diff --git a/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/RegisterOrderTest.java b/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/RegisterOrderTest.java index 90117a8aa..2e23dd731 100644 --- a/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/RegisterOrderTest.java +++ b/client/impl/src/test/java/com/alipay/sofa/registry/client/provider/RegisterOrderTest.java @@ -16,6 +16,20 @@ */ package com.alipay.sofa.registry.client.provider; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.alipay.sofa.registry.client.api.Publisher; import com.alipay.sofa.registry.client.api.Subscriber; import com.alipay.sofa.registry.client.api.SubscriberDataObserver; @@ -26,19 +40,6 @@ import com.alipay.sofa.registry.core.model.DataBox; import com.alipay.sofa.registry.core.model.PublisherRegister; import com.alipay.sofa.registry.core.model.SubscriberRegister; -import org.junit.After; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @@ -126,7 +127,7 @@ public void publishAndUnregister() throws InterruptedException { // step 1 Publisher publisher = registryClient.register(new PublisherRegistration(dataId), data); - Thread.sleep(500L); + Thread.sleep(2000L); String registId = publisher.getRegistId(); PublisherRegister publisherRegister = mockServer.queryPubliser(registId); @@ -137,7 +138,7 @@ public void publishAndUnregister() throws InterruptedException { // step 2 publisher.unregister(); - Thread.sleep(500L); + Thread.sleep(2000L); assertNull(mockServer.queryPubliser(registId)); @@ -151,7 +152,7 @@ public void publishAndRefused() throws InterruptedException { // step 1 Publisher publisher = registryClient.register(new PublisherRegistration(data), data); - Thread.sleep(500L); + Thread.sleep(2000L); String registId = publisher.getRegistId(); PublisherRegister publisherRegister = mockServer.queryPubliser(registId); @@ -182,7 +183,7 @@ public void handleData(String dataId, UserData data) { Subscriber subscriber = registryClient.register(new SubscriberRegistration(dataId, dataObserver)); - Thread.sleep(500L); + Thread.sleep(2000L); String registId = subscriber.getRegistId(); SubscriberRegister subscriberRegister = mockServer.querySubscriber(registId); @@ -191,7 +192,7 @@ public void handleData(String dataId, UserData data) { // step 2 subscriber.unregister(); - Thread.sleep(500L); + Thread.sleep(2000L); assertNull(mockServer.queryPubliser(registId)); @@ -211,7 +212,7 @@ public void handleData(String dataId, UserData data) { Subscriber subscriber = registryClient.register(new SubscriberRegistration( "subscribeAndRefused", dataObserver)); - Thread.sleep(500L); + Thread.sleep(2000L); String registId = subscriber.getRegistId(); SubscriberRegister subscriberRegister = mockServer.querySubscriber(registId); diff --git a/client/log/pom.xml b/client/log/pom.xml index b41fb3f47..546462dbb 100644 --- a/client/log/pom.xml +++ b/client/log/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index ea36b1f84..1bef3c48c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -7,7 +7,7 @@ com.alipay.sofa registry-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index e01707058..39b8e57ba 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 8d1e42280..8d0e95d58 100644 --- a/pom.xml +++ b/pom.xml @@ -2,11 +2,11 @@ + 4.0.0 - com.alipay.sofa registry-parent - 5.2.0-SNAPSHOT + 5.4.5 pom ${project.groupId}:${project.artifactId} @@ -59,7 +59,7 @@ 1.8 1.8 UTF-8 - 1.5.2 + 1.6.2 2.6 18.0 1.0.12 @@ -67,24 +67,30 @@ 5.0.2 ${junit.version}.2 1.0.2 - 3.3.6 - 4.1.25.Final + 3.3.8 + 4.1.42.Final 2.26 3.5.1 1.7.21 1.5.2 1.10.19 1.6.6 - 1.2.4 + 1.3.5.Alpha1 4.0.2 2.4 - 9.4.12.v20180830 + 9.4.19.v20190610 + 6.4.6 ${user.dir} -Dnetwork_interface_denylist=docker0 + + org.slf4j + jul-to-slf4j + 1.7.9 + org.springframework.boot spring-boot-starter-parent @@ -307,6 +313,12 @@ ${metrics.version} + + commons-collections + commons-collections + 3.2.2 + + junit @@ -332,6 +344,11 @@ ${powermock.version} test + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + @@ -396,6 +413,9 @@ org.apache.maven.plugins maven-surefire-plugin 2.21.0 + + once + org.jacoco diff --git a/server/common/model/pom.xml b/server/common/model/pom.xml index d87c55c92..49734dd14 100644 --- a/server/common/model/pom.xml +++ b/server/common/model/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-common - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/CommonResponse.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/CommonResponse.java index e85455adf..72f7d5591 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/CommonResponse.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/CommonResponse.java @@ -108,4 +108,16 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CommonResponse{"); + sb.append("success=").append(success); + sb.append(", message='").append(message).append('\''); + sb.append('}'); + return sb.toString(); + } } \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/DatumSnapshotRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/DatumSnapshotRequest.java new file mode 100644 index 000000000..0b517fa61 --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/DatumSnapshotRequest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model; + +import java.io.Serializable; +import java.util.List; + +import com.alipay.sofa.registry.common.model.store.Publisher; + +/** + * + * @author shangyu.wh + * @version $Id: DatumSnapshotRequest.java, v 0.1 2019-05-30 11:09 shangyu.wh Exp $ + */ +public class DatumSnapshotRequest implements Serializable { + + private static final long serialVersionUID = 2193212935059863551L; + + private final String connectId; + + private final String dataServerIp; + + private final List publishers; + + public DatumSnapshotRequest(String connectId, String dataServerIp, List publishers) { + this.connectId = connectId; + this.dataServerIp = dataServerIp; + this.publishers = publishers; + } + + /** + * Getter method for property connectId. + * + * @return property value of connectId + */ + public String getConnectId() { + return connectId; + } + + /** + * Getter method for property dataServerIp. + * + * @return property value of dataServerIp + */ + public String getDataServerIp() { + return dataServerIp; + } + + /** + * Getter method for property publishers. + * + * @return property value of publishers + */ + public List getPublishers() { + return publishers; + } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DatumSnapshotRequest{"); + sb.append("connectId='").append(connectId).append('\''); + sb.append(", dataServerIp='").append(dataServerIp).append('\''); + sb.append(", publishers.size=").append(publishers.size()); + sb.append('}'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/PublisherDigestUtil.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/PublisherDigestUtil.java new file mode 100644 index 000000000..67ad72236 --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/PublisherDigestUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model; + +import java.util.Collection; + +import com.alipay.sofa.registry.common.model.store.Publisher; + +/** + * + * @author kezhu.wukz + * @author shangyu.wh + * @version $Id: PublisherDigestUtil.java, v 0.1 2019-05-30 20:58 shangyu.wh Exp $ + */ +public class PublisherDigestUtil { + + public static long getDigestValueSum(Collection publishers) { + long digest = 0L; + if (publishers != null && !publishers.isEmpty()) { + for (Publisher publisher : publishers) { + digest += getDigestValue(publisher); + } + } + return digest; + } + + public static long getDigestValue(Publisher publisher) { + String registerId = publisher.getRegisterId(); + Long version = publisher.getVersion(); + long registerTimestamp = publisher.getRegisterTimestamp(); + long result = registerId != null ? registerId.hashCode() : 0; + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (int) (registerTimestamp ^ (registerTimestamp >>> 32)); + return result; + } +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/RenewDatumRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/RenewDatumRequest.java new file mode 100644 index 000000000..603c40fb3 --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/RenewDatumRequest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model; + +import java.io.Serializable; + +/** + * + * @author shangyu.wh + * @author kezhu.wukz + * @version $Id: RenewRequest.java, v 0.1 2019-05-30 10:58 shangyu.wh Exp $ + */ +public class RenewDatumRequest implements Serializable { + + private static final long serialVersionUID = 683097441984338311L; + + private final String connectId; + + private final String dataServerIP; + + private final String digestSum; + + public RenewDatumRequest(String connectId, String dataServerIP, String digestSum) { + this.connectId = connectId; + this.dataServerIP = dataServerIP; + this.digestSum = digestSum; + } + + /** + * Getter method for property connectId. + * + * @return property value of connectId + */ + public String getConnectId() { + return connectId; + } + + /** + * Getter method for property dataServerIP. + * + * @return property value of dataServerIP + */ + public String getDataServerIP() { + return dataServerIP; + } + + /** + * Getter method for property digestSum. + * + * @return property value of digestSum + */ + public String getDigestSum() { + return digestSum; + } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RenewDatumRequest{"); + sb.append("connectId='").append(connectId).append('\''); + sb.append(", dataServerIP='").append(dataServerIP).append('\''); + sb.append(", digestSum='").append(digestSum).append('\''); + sb.append('}'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/ServerDataBox.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/ServerDataBox.java index 75dc51877..f65d8b942 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/ServerDataBox.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/ServerDataBox.java @@ -16,8 +16,6 @@ */ package com.alipay.sofa.registry.common.model; -import com.fasterxml.jackson.annotation.JsonIgnore; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,6 +23,8 @@ import java.io.ObjectOutputStream; import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * * @author zhuoyu.sjw @@ -93,7 +93,7 @@ public Object getObject() { * @throws ClassNotFoundException the class not found exception */ public Object extract() throws IOException, ClassNotFoundException { - if (isInBytes()) { + if (object == null && isInBytes()) { ByteArrayInputStream bis = new ByteArrayInputStream(bytes); if (serialization != SERIALIZED_BY_JAVA) { throw new IOException("Unsupported serialization type: " + serialization); diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/constants/ValueConstants.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/constants/ValueConstants.java index bd6f20409..cd0ed73d1 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/constants/ValueConstants.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/constants/ValueConstants.java @@ -23,37 +23,61 @@ */ public class ValueConstants { + /** + * connectId: sourceAddress_targetAddress + */ + public static final String CONNECT_ID_SPLIT = "_"; + /** * The constant DEFAULT_GROUP. */ - public static final String DEFAULT_GROUP = "DEFAULT_GROUP"; + public static final String DEFAULT_GROUP = "DEFAULT_GROUP"; /** * The constant DEFAULT_ZONE. */ - public static final String DEFAULT_ZONE = "DEFAULT_ZONE"; + public static final String DEFAULT_ZONE = "DEFAULT_ZONE"; - public static final String DEFAULT_INSTANCE_ID = "DEFAULT_INSTANCE_ID"; + public static final String DEFAULT_INSTANCE_ID = "DEFAULT_INSTANCE_ID"; /** * The constant DEFAULT_DATA_CENTER. */ - public static final String DEFAULT_DATA_CENTER = "DefaultDataCenter"; + public static final String DEFAULT_DATA_CENTER = "DefaultDataCenter"; + + public static final long DEFAULT_NO_DATUM_VERSION = 1L; + + private static final Integer SYSTEM_RAFT_PORT = Integer + .getInteger("RAFT_SERVER_PORT"); + + public static final int RAFT_SERVER_PORT = SYSTEM_RAFT_PORT != null ? SYSTEM_RAFT_PORT + : 9614; - public static final long DEFAULT_NO_DATUM_VERSION = 1L; + private static final String SYSTEM_RAFT_GROUP = System + .getProperty("RAFT_SERVER_GROUP"); - private static final Integer SYSTEM_RAFT_PORT = Integer - .getInteger("RAFT_SERVER_PORT"); + public static final String RAFT_SERVER_GROUP = SYSTEM_RAFT_GROUP != null ? SYSTEM_RAFT_GROUP + : "RegistryGroup"; - public static final int RAFT_SERVER_PORT = SYSTEM_RAFT_PORT != null ? SYSTEM_RAFT_PORT - : 9614; + public static final String STOP_PUSH_DATA_SWITCH_DATA_ID = "session.stop.push.data.switch#@#9600#@#CONFIG"; - private static final String SYSTEM_RAFT_GROUP = System - .getProperty("RAFT_SERVER_GROUP"); + public static final String BLACK_LIST_DATA_ID = "session.blacklist.data#@#9600#@#CONFIG"; - public static final String RAFT_SERVER_GROUP = SYSTEM_RAFT_GROUP != null ? SYSTEM_RAFT_GROUP - : "RegistryGroup"; + public static final String ENABLE_DATA_RENEW_SNAPSHOT = "session.enable.datum.renew.switch#@#9600#@#CONFIG"; - public static final String STOP_PUSH_DATA_SWITCH_DATA_ID = "session.stop.push.data.switch#@#9600#@#CONFIG"; + public static final String ENABLE_DATA_DATUM_EXPIRE = "data.enable.datum.expire.switch#@#9600#@#CONFIG"; + + public static final String LOGGER_NAME_RENEW = "RENEW-LOGGER"; + + /** + * switch key for dataId sensitive is disable or not + */ + public static final String DISABLE_DATA_ID_CASE_SENSITIVE_SWITCH = "disable.dataId.case.sensitive"; + /** + * switch for dataId sensitive is disable or not, default value is false which means dataId is case sensitive + */ + public static final Boolean DISABLE_DATA_ID_CASE_SENSITIVE = Boolean + .valueOf(System + .getProperty(DISABLE_DATA_ID_CASE_SENSITIVE_SWITCH)); } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/Datum.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/Datum.java index 61044ea8f..80d0b9e0c 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/Datum.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/Datum.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.common.model.dataserver; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.common.model.store.WordCache; - import java.io.Serializable; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.util.DatumVersionUtil; + /** * datum store in dataserver * @@ -61,7 +62,7 @@ public Datum() { * @param dataCenter */ public Datum(String dataInfoId, String dataCenter) { - this.dataInfoId = dataInfoId; + this.dataInfoId = WordCache.getInstance().getWordCache(dataInfoId); this.dataCenter = WordCache.getInstance().getWordCache(dataCenter); updateVersion(); } @@ -96,7 +97,7 @@ public Datum(Publisher publisher, String dataCenter, long version) { } public void updateVersion() { - this.version = System.currentTimeMillis(); + this.version = DatumVersionUtil.nextId(); } /** @@ -243,7 +244,7 @@ public void setContainsUnPub(boolean containsUnPub) { this.containsUnPub = containsUnPub; } - public static Datum processDatum(Datum datum) { + public static Datum internDatum(Datum datum) { datum.setDataCenter(datum.getDataCenter()); datum.setDataInfoId(datum.getDataInfoId()); datum.setDataId(datum.getDataId()); @@ -252,7 +253,13 @@ public static Datum processDatum(Datum datum) { Map pubMap = datum.getPubMap(); if (pubMap != null && !pubMap.isEmpty()) { - pubMap.forEach((registerId, publisher) -> Publisher.processPublisher(publisher)); + pubMap.forEach((registerId, publisher) -> { + // let registerId == pub.getRegisterId in every , for reducing old gen memory + // because this Datum is put into Memory directly, by DatumCache.coverDatum + publisher.setRegisterId(registerId); + // change publisher word cache + Publisher.internPublisher(publisher); + }); } return datum; diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/SessionServerRegisterRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/SessionServerRegisterRequest.java index 16f00f56b..b852aa3ef 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/SessionServerRegisterRequest.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/SessionServerRegisterRequest.java @@ -30,16 +30,16 @@ public class SessionServerRegisterRequest implements Serializable { private String processId; - private Set clientHosts; + private Set connectIds; /** * constructor * @param processId - * @param clientHosts + * @param connectIds */ - public SessionServerRegisterRequest(String processId, Set clientHosts) { + public SessionServerRegisterRequest(String processId, Set connectIds) { this.processId = processId; - this.clientHosts = clientHosts; + this.connectIds = connectIds; } /** @@ -61,17 +61,17 @@ public void setProcessId(String processId) { } /** - * Getter method for property clientHosts. + * Getter method for property connectIds. * - * @return property value of clientHosts + * @return property value of connectIds */ - public Set getClientHosts() { - return clientHosts; + public Set getConnectIds() { + return connectIds; } @Override public String toString() { return new StringBuilder("[SessionServerRegisterRequest] processId=") - .append(this.processId).append(", clientHosts=").append(this.clientHosts).toString(); + .append(this.processId).append(", connectIds=").append(this.connectIds).toString(); } } \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/UnPublishDataRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/UnPublishDataRequest.java index cfa8554ac..33fbdbf0c 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/UnPublishDataRequest.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/UnPublishDataRequest.java @@ -34,12 +34,6 @@ public class UnPublishDataRequest implements Serializable { private long registerTimestamp; - /** - * constructor - */ - public UnPublishDataRequest() { - } - /** * construtor * @param dataInfoId diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ConfigureLoadbalanceRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ConfigureLoadbalanceRequest.java new file mode 100644 index 000000000..b430b16fb --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ConfigureLoadbalanceRequest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model.metaserver; + +import java.io.Serializable; +import java.util.Map; + +/** + * @author xiangxu + * @version : LoadbalanceConfig.java, v 0.1 2020年05月29日 10:19 上午 xiangxu Exp $ + */ +public class ConfigureLoadbalanceRequest implements Serializable { + private int maxConnections; + + public ConfigureLoadbalanceRequest(int maxConnections) { + this.maxConnections = maxConnections; + } + + public int getMaxConnections() { + return maxConnections; + } + + public void setMaxConnections(int maxConnections) { + this.maxConnections = maxConnections; + } + +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/DataNode.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/DataNode.java index ca3bba378..e5e623f63 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/DataNode.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/DataNode.java @@ -213,10 +213,6 @@ public void setRegistrationTimestamp(long registrationTimestamp) { public String toString() { final StringBuilder sb = new StringBuilder("DataNode{"); sb.append("ip=").append(getIp()); - sb.append(", dataCenter='").append(dataCenter).append('\''); - sb.append(", regionId='").append(regionId).append('\''); - sb.append(", nodeStatus=").append(nodeStatus); - sb.append(", registrationTimestamp=").append(registrationTimestamp); sb.append('}'); return sb.toString(); } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/GetLoadbalanceMetricsRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/GetLoadbalanceMetricsRequest.java new file mode 100644 index 000000000..ad781aff8 --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/GetLoadbalanceMetricsRequest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model.metaserver; + +import java.io.Serializable; + +/** + * @author xiangxu + * @version : SessionLoadbalanceMetrics.java, v 0.1 2020年05月27日 11:13 上午 xiangxu Exp $ + */ +public class GetLoadbalanceMetricsRequest implements Serializable { +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/LoadbalanceMetrics.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/LoadbalanceMetrics.java new file mode 100644 index 000000000..42a31a8b6 --- /dev/null +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/LoadbalanceMetrics.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.common.model.metaserver; + +import java.io.Serializable; + +/** + * @author xiangxu + * @version : LoadbalanceMetrics.java, v 0.1 2020年05月27日 2:29 下午 xiangxu Exp $ + */ +public class LoadbalanceMetrics implements Serializable { + private int connectionCount; + + public LoadbalanceMetrics() { + } + + public int getConnectionCount() { + return connectionCount; + } + + public void setConnectionCount(int connectionCount) { + this.connectionCount = connectionCount; + } +} \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/MetaNode.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/MetaNode.java index 6adf2457c..01d107cb0 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/MetaNode.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/MetaNode.java @@ -131,10 +131,7 @@ public void setRegionId(String regionId) { @Override public String toString() { final StringBuilder sb = new StringBuilder("MetaNode{"); - sb.append("nodeUrl=").append(getIp()); - sb.append(", dataCenter='").append(dataCenter).append('\''); - sb.append(", regionId='").append(regionId).append('\''); - sb.append(", nodeStatus=").append(nodeStatus); + sb.append("ip=").append(getIp()); sb.append('}'); return sb.toString(); } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/NotifyProvideDataChange.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/NotifyProvideDataChange.java index ad14df8f0..3fa7f078c 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/NotifyProvideDataChange.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/NotifyProvideDataChange.java @@ -17,6 +17,10 @@ package com.alipay.sofa.registry.common.model.metaserver; import java.io.Serializable; +import java.util.Set; + +import com.alipay.sofa.registry.common.model.Node.NodeType; +import com.google.common.collect.Sets; /** * @@ -25,11 +29,13 @@ */ public class NotifyProvideDataChange implements Serializable { - private String dataInfoId; + private String dataInfoId; + + private Long version; - private Long version; + private DataOperator dataOperator; - private DataOperator dataOperator; + private Set nodeTypes; /** * constructor @@ -38,9 +44,15 @@ public class NotifyProvideDataChange implements Serializable { * @param dataOperator */ public NotifyProvideDataChange(String dataInfoId, Long version, DataOperator dataOperator) { + this(dataInfoId, version, dataOperator, Sets.newHashSet(NodeType.SESSION)); + } + + public NotifyProvideDataChange(String dataInfoId, Long version, DataOperator dataOperator, + Set nodeTypes) { this.dataInfoId = dataInfoId; this.version = version; this.dataOperator = dataOperator; + this.nodeTypes = nodeTypes; } /** @@ -97,12 +109,31 @@ public void setDataOperator(DataOperator dataOperator) { this.dataOperator = dataOperator; } + /** + * Getter method for property nodeType. + * + * @return property value of nodeType + */ + public Set getNodeTypes() { + return nodeTypes; + } + + /** + * Setter method for property nodeType. + * + * @param nodeTypes value to be assigned to property nodeType + */ + public void setNodeTypes(Set nodeTypes) { + this.nodeTypes = nodeTypes; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("NotifyProvideDataChange{"); sb.append("dataInfoId='").append(dataInfoId).append('\''); sb.append(", version=").append(version); sb.append(", dataOperator=").append(dataOperator); + sb.append(", nodeTypes=").append(nodeTypes); sb.append('}'); return sb.toString(); } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ReNewNodesRequest.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/RenewNodesRequest.java similarity index 92% rename from server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ReNewNodesRequest.java rename to server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/RenewNodesRequest.java index 50485e65c..0794e5152 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/ReNewNodesRequest.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/RenewNodesRequest.java @@ -16,16 +16,16 @@ */ package com.alipay.sofa.registry.common.model.metaserver; -import com.alipay.sofa.registry.common.model.Node; - import java.io.Serializable; +import com.alipay.sofa.registry.common.model.Node; + /** * * @author shangyu.wh * @version $Id: RenewNodesRequest.java, v 0.1 2018-03-30 19:51 shangyu.wh Exp $ */ -public class ReNewNodesRequest implements Serializable { +public class RenewNodesRequest implements Serializable { private int duration; @@ -35,7 +35,7 @@ public class ReNewNodesRequest implements Serializable { * constructor * @param node */ - public ReNewNodesRequest(T node) { + public RenewNodesRequest(T node) { this.node = node; } @@ -68,7 +68,7 @@ public T getNode() { @Override public String toString() { - final StringBuilder sb = new StringBuilder("ReNewNodesRequest{"); + final StringBuilder sb = new StringBuilder("RenewNodesRequest{"); sb.append("duration=").append(duration); sb.append(", node=").append(node); sb.append('}'); diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/SessionNode.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/SessionNode.java index d6beeaf67..8c9a3487e 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/SessionNode.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/metaserver/SessionNode.java @@ -55,6 +55,14 @@ public URL getNodeUrl() { return nodeUrl; } + /** + * get ip address from nodeUrl + * @return + */ + public String getIp() { + return nodeUrl == null ? "" : nodeUrl.getIpAddress(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -167,10 +175,7 @@ public void setNodeStatus(NodeStatus nodeStatus) { @Override public String toString() { final StringBuilder sb = new StringBuilder("SessionNode{"); - sb.append("nodeUrl=").append(nodeUrl); - sb.append(", regionId='").append(regionId).append('\''); - sb.append(", name='").append(name).append('\''); - sb.append(", nodeStatus=").append(nodeStatus); + sb.append("ip=").append(getIp()); sb.append('}'); return sb.toString(); } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/BaseInfo.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/BaseInfo.java index 7c4983356..e2de744bc 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/BaseInfo.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/BaseInfo.java @@ -16,12 +16,12 @@ */ package com.alipay.sofa.registry.common.model.store; -import com.fasterxml.jackson.annotation.JsonIgnore; - import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnore; + /** * * @author shangyu.wh @@ -51,6 +51,8 @@ public abstract class BaseInfo implements Serializable, StoreData { private URL sourceAddress; + private URL targetAddress; + private ClientVersion clientVersion; private String group; @@ -136,7 +138,7 @@ public String getProcessId() { * @param processId value to be assigned to property processId */ public void setProcessId(String processId) { - this.processId = processId; + this.processId = WordCache.getInstance().getWordCache(processId); } /** @@ -175,6 +177,24 @@ public void setSourceAddress(URL sourceAddress) { this.sourceAddress = sourceAddress; } + /** + * Getter method for property targetAddress. + * + * @return property value of targetAddress + */ + public URL getTargetAddress() { + return targetAddress; + } + + /** + * Setter method for property targetAddress. + * + * @param targetAddress value to be assigned to property targetAddress + */ + public void setTargetAddress(URL targetAddress) { + this.targetAddress = targetAddress; + } + /** * Getter method for property attributes. * @@ -193,7 +213,7 @@ public void setAttributes(Map attributes) { Map newAttributes = new HashMap<>(); if (attributes != null && !attributes.isEmpty()) { attributes.forEach((key, value) -> newAttributes - .put(WordCache.getInstance().getWordCache(key), value)); + .put(key, value)); } this.attributes = newAttributes; } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/DataInfo.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/DataInfo.java index f519b44b0..02e5ee7f2 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/DataInfo.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/DataInfo.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.common.model.store; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; + import java.io.Serializable; /** @@ -61,7 +63,11 @@ public static String toDataInfoId(String dataId, String instanceId, String dataT if (dataId == null || dataId.isEmpty()) { throw new IllegalArgumentException("error dataId:" + dataId); } - buf.append(dataId); + if (ValueConstants.DISABLE_DATA_ID_CASE_SENSITIVE) { + buf.append(dataId.toUpperCase()); + } else { + buf.append(dataId); + } if (instanceId == null || instanceId.isEmpty()) { throw new IllegalArgumentException("error instanceId:" + instanceId); diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Publisher.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Publisher.java index 8ecd7566b..a216732aa 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Publisher.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Publisher.java @@ -16,12 +16,12 @@ */ package com.alipay.sofa.registry.common.model.store; +import java.util.List; + import com.alipay.sofa.registry.common.model.PublishType; import com.alipay.sofa.registry.common.model.ServerDataBox; import com.fasterxml.jackson.annotation.JsonIgnore; -import java.util.List; - /** * * @author shangyu.wh @@ -90,8 +90,8 @@ protected String getOtherInfo() { * @param publisher * @return */ - public static Publisher processPublisher(Publisher publisher) { - + public static Publisher internPublisher(Publisher publisher) { + publisher.setRegisterId(publisher.getRegisterId()); publisher.setDataInfoId(publisher.getDataInfoId()); publisher.setInstanceId(publisher.getInstanceId()); publisher.setGroup(publisher.getGroup()); @@ -101,13 +101,6 @@ public static Publisher processPublisher(Publisher publisher) { publisher.setProcessId(publisher.getProcessId()); publisher.setAppName(publisher.getAppName()); - if (publisher.getSourceAddress() != null) { - publisher.setSourceAddress(new URL(publisher.getSourceAddress().getIpAddress(), - publisher.getSourceAddress().getPort())); - } - - publisher.setAttributes(publisher.getAttributes()); - return publisher; } diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/StoreData.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/StoreData.java index 1ec2d9a66..5746a45f8 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/StoreData.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/StoreData.java @@ -27,7 +27,7 @@ public interface StoreData { * DataType enum */ enum DataType { - SUBSCRIBER, PUBLISHER, WATCHER, UNPUBLISHER + SUBSCRIBER, PUBLISHER, WATCHER, UN_PUBLISHER } /** diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Subscriber.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Subscriber.java index 2f0b8b78d..3d9b532b7 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Subscriber.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Subscriber.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.common.model.store; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import com.alipay.sofa.registry.common.model.ElementType; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.core.model.ScopeEnum; import com.fasterxml.jackson.annotation.JsonIgnore; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - /** * * @author shangyu.wh @@ -31,16 +32,16 @@ public class Subscriber extends BaseInfo { /** UID */ - private static final long serialVersionUID = 98433360274932292L; + private static final long serialVersionUID = 98433360274932292L; /** */ - private ScopeEnum scope; + private ScopeEnum scope; /** */ - private ElementType elementType; + private ElementType elementType; /** - * all dataCenter push dataInfo version + * last push context */ - private Map lastPushVersions = new ConcurrentHashMap<>(); + private Map lastPushContexts = new ConcurrentHashMap<>(); /** * Getter method for property scope. @@ -71,7 +72,11 @@ public ElementType getElementType() { */ public boolean checkVersion(String dataCenter, Long version) { - Long oldVersion = lastPushVersions.get(dataCenter); + PushContext lastPushContext = lastPushContexts.get(dataCenter); + if (lastPushContext == null) { + return version != null; + } + Long oldVersion = lastPushContext.pushVersion; if (oldVersion == null) { return version != null; } else { @@ -88,15 +93,26 @@ public boolean checkVersion(String dataCenter, Long version) { * @return */ public void checkAndUpdateVersion(String dataCenter, Long version) { + checkAndUpdateVersion(dataCenter, version, -1); + } + + /** + * check version input greater or equal to current version + * @param version + * @return + */ + public void checkAndUpdateVersion(String dataCenter, Long version, int pubCount) { while (true) { - Long oldVersion = lastPushVersions.putIfAbsent(dataCenter, version); + PushContext pushContext = new PushContext(version, pubCount); + PushContext oldPushContext = lastPushContexts.putIfAbsent(dataCenter, pushContext); // Add firstly - if (oldVersion == null) { + if (oldPushContext == null) { break; } else { - if (version > oldVersion) { - if (lastPushVersions.replace(dataCenter, oldVersion, version)) { + if (oldPushContext.pushVersion == null + || (pushContext.pushVersion != null && pushContext.pushVersion > oldPushContext.pushVersion)) { + if (lastPushContexts.replace(dataCenter, oldPushContext, pushContext)) { break; } } else { @@ -106,6 +122,23 @@ public void checkAndUpdateVersion(String dataCenter, Long version) { } } + /** + * If the pushed data is empty, check the last push, for avoid continuous empty datum push + */ + public boolean allowPush(String dataCenter, int pubCount) { + boolean allowPush = true; + // condition of no push: + // 1. last push count is 0 and this time is also 0 + // 2. last push is a valid push (version > 1) + if (pubCount == 0) { + PushContext pushContext = lastPushContexts.get(dataCenter); + allowPush = !(pushContext != null && pushContext.pushPubCount == 0 + //last push is a valid push + && pushContext.pushVersion != null && pushContext.pushVersion > ValueConstants.DEFAULT_NO_DATUM_VERSION); + } + return allowPush; + } + /** * Setter method for property elementType. * @@ -126,28 +159,10 @@ protected String getOtherInfo() { final StringBuilder sb = new StringBuilder("scope="); sb.append(scope).append(","); sb.append("elementType=").append(elementType).append(","); - sb.append("lastPushVersion=").append(lastPushVersions); + sb.append("pushVersion=").append(lastPushContexts); return sb.toString(); } - /** - * Getter method for property lastPushVersions. - * - * @return property value of lastPushVersions - */ - public Map getLastPushVersions() { - return lastPushVersions; - } - - /** - * Setter method for property lastPushVersions . - * - * @param lastPushVersions value to be assigned to property lastPushVersions - */ - public void setLastPushVersions(Map lastPushVersions) { - this.lastPushVersions = lastPushVersions; - } - /** * @see Object#toString() */ @@ -156,9 +171,57 @@ public String toString() { final StringBuilder sb = new StringBuilder("Subscriber{"); sb.append("scope=").append(scope); sb.append(", elementType=").append(elementType); - sb.append(", lastPushVersions=").append(lastPushVersions); + sb.append(", lastPushContexts=").append(lastPushContexts); sb.append(", super=").append(super.toString()); sb.append('}'); return sb.toString(); } + + /** + * change subscriber word cache + * @param subscriber + * @return + */ + public static Subscriber internSubscriber(Subscriber subscriber) { + subscriber.setRegisterId(subscriber.getRegisterId()); + subscriber.setDataInfoId(subscriber.getDataInfoId()); + subscriber.setInstanceId(subscriber.getInstanceId()); + subscriber.setGroup(subscriber.getGroup()); + subscriber.setDataId(subscriber.getDataId()); + subscriber.setClientId(subscriber.getClientId()); + subscriber.setCell(subscriber.getCell()); + subscriber.setProcessId(subscriber.getProcessId()); + subscriber.setAppName(subscriber.getAppName()); + + return subscriber; + } + + static class PushContext { + /** + * last pushed dataInfo version + */ + private Long pushVersion; + + /** + * push pushed dataInfo pubCount + */ + private int pushPubCount; + + public PushContext(Long pushVersion, int pushPubCount) { + this.pushVersion = pushVersion; + this.pushPubCount = pushPubCount; + } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("PushContext{"); + sb.append("pushVersion=").append(pushVersion); + sb.append(", pushPubCount=").append(pushPubCount); + sb.append('}'); + return sb.toString(); + } + } } \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Watcher.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Watcher.java index 6ffa610f9..f369d5e4e 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Watcher.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/Watcher.java @@ -27,4 +27,23 @@ public class Watcher extends BaseInfo { public DataType getDataType() { return DataType.WATCHER; } + + /** + * change watcher word cache + * @param watcher + * @return + */ + public static Watcher internWatcher(Watcher watcher) { + watcher.setRegisterId(watcher.getRegisterId()); + watcher.setDataInfoId(watcher.getDataInfoId()); + watcher.setInstanceId(watcher.getInstanceId()); + watcher.setGroup(watcher.getGroup()); + watcher.setDataId(watcher.getDataId()); + watcher.setClientId(watcher.getClientId()); + watcher.setCell(watcher.getCell()); + watcher.setProcessId(watcher.getProcessId()); + watcher.setAppName(watcher.getAppName()); + + return watcher; + } } \ No newline at end of file diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/WordCache.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/WordCache.java index 2734c8ac0..f00e3c938 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/WordCache.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/store/WordCache.java @@ -16,7 +16,8 @@ */ package com.alipay.sofa.registry.common.model.store; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; /** * @@ -45,7 +46,7 @@ public static WordCache getInstance() { /** * word cache map */ - private ConcurrentHashMap map = new ConcurrentHashMap<>(); + private Interner interners = Interners.newWeakInterner(); /** * @@ -56,8 +57,7 @@ public String getWordCache(String s) { if (s == null) { return null; } - String oldValue = map.putIfAbsent(s, s); - return oldValue == null ? s : oldValue; + return interners.intern(s); } } \ No newline at end of file diff --git a/server/common/pom.xml b/server/common/pom.xml index 9027270dc..40b568ca7 100644 --- a/server/common/pom.xml +++ b/server/common/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/common/util/pom.xml b/server/common/util/pom.xml index 827af0b6e..fd262ae30 100644 --- a/server/common/util/pom.xml +++ b/server/common/util/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-common - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/ReporterUtils.java b/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/ReporterUtils.java index fb5f24e36..7cc79bc33 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/ReporterUtils.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/ReporterUtils.java @@ -16,12 +16,13 @@ */ package com.alipay.sofa.registry.metrics; +import java.util.concurrent.TimeUnit; + import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Slf4jReporter; -import java.util.concurrent.TimeUnit; - /** * * @author shangyu.wh @@ -29,6 +30,8 @@ */ public class ReporterUtils { + private static final Logger METRIC_LOGGER = LoggerFactory.getLogger("REGISTRY-METRICS"); + /** * start slf4j reporter * @param period @@ -46,4 +49,13 @@ public static void startSlf4jReporter(long period, MetricRegistry registry, Logg } } + + /** + * start slf4j reporter + * @param period + * @param registry + */ + public static void startSlf4jReporter(long period, MetricRegistry registry) { + startSlf4jReporter(period, registry, METRIC_LOGGER); + } } \ No newline at end of file diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/TaskMetrics.java b/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/TaskMetrics.java index ae0ec816c..4408046de 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/TaskMetrics.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/metrics/TaskMetrics.java @@ -18,7 +18,13 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Sets; +import io.netty.util.internal.ConcurrentSet; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; /** @@ -29,6 +35,7 @@ public class TaskMetrics { private final MetricRegistry metrics; + private final Set executorNames = Sets.newConcurrentHashSet(); private TaskMetrics() { this.metrics = new MetricRegistry(); @@ -53,21 +60,45 @@ public MetricRegistry getMetricRegistry() { } public void registerThreadExecutor(String executorName, ThreadPoolExecutor executor) { + executorNames.add(executorName); metrics.register(MetricRegistry.name(executorName, "queue"), - (Gauge) () -> executor.getQueue().size()); + (Gauge) () -> executor.getQueue().size()); metrics.register(MetricRegistry.name(executorName, "current"), - (Gauge) executor::getPoolSize); + (Gauge) executor::getPoolSize); metrics.register(MetricRegistry.name(executorName, "active"), - (Gauge) executor::getActiveCount); + (Gauge) executor::getActiveCount); metrics.register(MetricRegistry.name(executorName, "completed"), - (Gauge) executor::getCompletedTaskCount); + (Gauge) executor::getCompletedTaskCount); metrics.register(MetricRegistry.name(executorName, "task"), - (Gauge) executor::getTaskCount); + (Gauge) executor::getTaskCount); + } + + public Set getExecutorNames() { + return executorNames; + } + public String metricsString() { + final String SYMBOLIC = " └─ "; + StringBuilder sb = new StringBuilder(); + sb.append("\n").append("ExecutorMetrics").append(" >>>>>>>>"); + sb.append("\n"); + for (String executorName : getExecutorNames()) { + MetricRegistry metricRegistry = getMetricRegistry(); + Map map = metricRegistry + .getGauges((name, value) -> name.startsWith(executorName)); + + sb.append(SYMBOLIC).append(executorName); + map.forEach((key, gauge) -> { + String name = key.substring(executorName.length() + 1); + sb.append(", ").append(name).append(":").append(gauge.getValue()); + }); + sb.append("\n"); + } + return sb.toString(); } } \ No newline at end of file diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/DefaultTaskListenerManager.java b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/DefaultTaskListenerManager.java index 8e6b23077..a0aa07b2f 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/DefaultTaskListenerManager.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/DefaultTaskListenerManager.java @@ -16,33 +16,35 @@ */ package com.alipay.sofa.registry.task.listener; -import java.util.ArrayList; import java.util.Collection; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + /** * @author xuanbei * @since 2018/12/28 */ public class DefaultTaskListenerManager implements TaskListenerManager { - private Collection taskListeners = new ArrayList<>(); + + private Multimap taskListeners = ArrayListMultimap.create(); @Override - public Collection getTaskListeners() { + public Multimap getTaskListeners() { return taskListeners; } @Override public void addTaskListener(TaskListener taskListener) { - taskListeners.add(taskListener); + taskListeners.put(taskListener.support(), taskListener); } @Override public void sendTaskEvent(TaskEvent taskEvent) { - + Collection taskListeners = this.taskListeners.get(taskEvent.getTaskType()); for (TaskListener taskListener : taskListeners) { - if (taskListener.support(taskEvent)) { - taskListener.handleEvent(taskEvent); - } + taskListener.handleEvent(taskEvent); } } } diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskEvent.java b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskEvent.java index 16f1abcc3..a46f40f8d 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskEvent.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskEvent.java @@ -16,10 +16,11 @@ */ package com.alipay.sofa.registry.task.listener; -import com.alipay.sofa.registry.task.TaskClosure; - import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alipay.sofa.registry.task.TaskClosure; /** * @@ -30,33 +31,41 @@ public class TaskEvent { public enum TaskType { //Session task - SUBSCRIBER_REGISTER_FETCH_TASK("SubscriberRegisterFetchTask"), SUBSCRIBER_PUSH_EMPTY_TASK( - "SubscriberPushEmptyTask"), WATCHER_REGISTER_FETCH_TASK( - "WatcherRegisterFetchTask"), DATA_CHANGE_FETCH_TASK( - "DataChangeFetchTask"), DATA_PUSH_TASK( - "DataPushTask"), DATA_CHANGE_FETCH_CLOUD_TASK( - "DataChangeFetchCloudTask"), RECEIVED_DATA_MULTI_PUSH_TASK( - "ReceivedDataMultiPushTask"), RECEIVED_DATA_CONFIG_PUSH_TASK( - "ReceivedDataConfigPushTask"), CANCEL_DATA_TASK( - "CancelDataTask"), SYNC_PUBLISHER_TASK( - "SyncPublisherTask"), SYNC_SUBSCRIBER_TASK( - "SyncSubscriberTask"), SESSION_REGISTER_DATA_TASK( - "SessionRegisterDataTask"), PROVIDE_DATA_CHANGE_FETCH_TASK( - "ProvideDataChangeFetchTask"), - - SUBSCRIBER_MULTI_FETCH_TASK("SubscriberMultiFetchTask"), + SUBSCRIBER_REGISTER_FETCH_TASK("SubscriberRegisterFetchTask"), // + SUBSCRIBER_PUSH_EMPTY_TASK("SubscriberPushEmptyTask"), // + WATCHER_REGISTER_FETCH_TASK("WatcherRegisterFetchTask"), // + DATA_CHANGE_FETCH_TASK("DataChangeFetchTask"), // + DATA_PUSH_TASK("DataPushTask"), // + DATA_CHANGE_FETCH_CLOUD_TASK("DataChangeFetchCloudTask"), // + RECEIVED_DATA_MULTI_PUSH_TASK("ReceivedDataMultiPushTask"), // + RECEIVED_DATA_CONFIG_PUSH_TASK("ReceivedDataConfigPushTask"), // + CANCEL_DATA_TASK("CancelDataTask"), // + SYNC_PUBLISHER_TASK("SyncPublisherTask"), // + SYNC_UNPUBLISHER_TASK("SyncUnPublisherTask"), // + SYNC_SUBSCRIBER_TASK("SyncSubscriberTask"), // + SYNC_UNSUBSCRIBER_TASK("SyncUnSubscriberTask"), // + SESSION_REGISTER_DATA_TASK("SessionRegisterDataTask"), // + PROVIDE_DATA_CHANGE_FETCH_TASK("ProvideDataChangeFetchTask"), // + SUBSCRIBER_MULTI_FETCH_TASK("SubscriberMultiFetchTask"), // + PUBLISH_DATA_TASK("PublishDataTask"), // + UN_PUBLISH_DATA_TASK("UnPublishDataTask"), // + RENEW_DATUM_TASK("RenewDatumTask"), // + DATUM_SNAPSHOT_TASK("DatumSnapshotTask"), // //Session Adapter task - USER_DATA_ELEMENT_PUSH_TASK("UserDataElementPushTask"), USER_DATA_ELEMENT_MULTI_PUSH_TASK( - "UserDataElementMultiPushTask"), + USER_DATA_ELEMENT_PUSH_TASK("UserDataElementPushTask"), // + USER_DATA_ELEMENT_MULTI_PUSH_TASK("UserDataElementMultiPushTask"), // //MetaServer task - SESSION_NODE_CHANGE_PUSH_TASK("SessionNodeChangePushTask"), DATA_NODE_CHANGE_PUSH_TASK( - "DataNodeChangePushTask"), RECEIVE_STATUS_CONFIRM_NOTIFY_TASK( - "ReceiveStatusConfirmNotifyTask"), PERSISTENCE_DATA_CHANGE_NOTIFY_TASK( - "PersistenceDataChangeNotifyTask"); + SESSION_NODE_CHANGE_PUSH_TASK("SessionNodeChangePushTask"), // + DATA_NODE_CHANGE_PUSH_TASK("DataNodeChangePushTask"), // + RECEIVE_STATUS_CONFIRM_NOTIFY_TASK("ReceiveStatusConfirmNotifyTask"), // + PERSISTENCE_DATA_CHANGE_NOTIFY_TASK("PersistenceDataChangeNotifyTask"), // + ; - private String name; + private String name; + + private AtomicInteger nextId = new AtomicInteger(0); TaskType(String name) { this.name = name; @@ -75,7 +84,11 @@ public String getName() { private long createTime; - private final Map attributes = new ConcurrentHashMap(); + private TaskClosure taskClosure; + + private final String taskId; + + private final Map attributes = new ConcurrentHashMap(); /** * constructor @@ -84,18 +97,27 @@ public String getName() { public TaskEvent(TaskType taskType) { this.taskType = taskType; this.createTime = System.currentTimeMillis(); + this.taskId = String.format("%s-%s-%s", taskType.name, this.createTime, + taskType.nextId.getAndIncrement()); } - private TaskClosure taskClosure; - /** * constructor * @param eventObj * @param taskType */ public TaskEvent(Object eventObj, TaskType taskType) { + this(taskType); this.eventObj = eventObj; - this.taskType = taskType; + } + + /** + * Getter method for property taskId. + * + * @return property value of taskId + */ + public String getTaskId() { + return taskId; } /** @@ -195,7 +217,12 @@ public long getCreateTime() { @Override public String toString() { - return "TaskEvent{" + "eventObj=" + eventObj + ", sendTimeStamp=" + sendTimeStamp - + ", attributes=" + attributes + '}'; + final StringBuilder sb = new StringBuilder("TaskEvent{"); + sb.append("eventObj=").append(eventObj); + sb.append(", sendTimeStamp=").append(sendTimeStamp); + sb.append(", attributes=").append(attributes); + sb.append(", taskId='").append(taskId).append('\''); + sb.append('}'); + return sb.toString(); } } \ No newline at end of file diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListener.java b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListener.java index 547d3b154..5601031eb 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListener.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.task.listener; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; + /** * * @author shangyu.wh @@ -26,10 +28,9 @@ public interface TaskListener { /** * com.alipay.sofa.registry.server.meta.listener type check * - * @param event - * @return true or false。 + * @return type */ - boolean support(TaskEvent event); + TaskType support(); /** * event execute diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListenerManager.java b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListenerManager.java index dc19c661d..58be148f8 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListenerManager.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/task/listener/TaskListenerManager.java @@ -16,7 +16,8 @@ */ package com.alipay.sofa.registry.task.listener; -import java.util.Collection; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.google.common.collect.Multimap; /** * @@ -29,7 +30,7 @@ public interface TaskListenerManager { * * @return */ - Collection getTaskListeners(); + Multimap getTaskListeners(); /** * diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimer.java b/server/common/util/src/main/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimer.java index f6cd4d206..7119a88a8 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimer.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimer.java @@ -16,45 +16,66 @@ */ package com.alipay.sofa.registry.timer; -import io.netty.util.HashedWheelTimer; -import io.netty.util.Timeout; -import io.netty.util.TimerTask; - import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timeout; +import io.netty.util.TimerTask; + /** * based on HashedWheelTimer, add function: exec TimerTask async * - * @author kezhu.wukz + * @author kezhu.wukz * @version $Id: AsyncHashedWheelTimer.java, v 0.1 2019-01-11 10:54 AM kezhu.wukz Exp $ */ public class AsyncHashedWheelTimer extends HashedWheelTimer { /** */ - private final Executor executor; + protected final Executor executor; /** */ - private final TaskFailedCallback taskFailedCallback; + protected final TaskFailedCallback taskFailedCallback; /** - * * @param threadFactory * @param tickDuration * @param unit * @param ticksPerWheel + * @param threadSize + * @param queueSize * @param asyncThreadFactory */ public AsyncHashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit, - int ticksPerWheel, ThreadFactory asyncThreadFactory, + int ticksPerWheel, int threadSize, int queueSize, + ThreadFactory asyncThreadFactory, TaskFailedCallback taskFailedCallback) { super(threadFactory, tickDuration, unit, ticksPerWheel); - this.executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, - new SynchronousQueue<>(), asyncThreadFactory); + ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadSize, threadSize, + 300L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(queueSize), asyncThreadFactory); + threadPoolExecutor.allowCoreThreadTimeOut(true); + this.executor = threadPoolExecutor; + this.taskFailedCallback = taskFailedCallback; + } + + /** + * + * @param threadFactory + * @param tickDuration + * @param unit + * @param ticksPerWheel + * @param asyncExecutor + */ + public AsyncHashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit, + int ticksPerWheel, Executor asyncExecutor, + TaskFailedCallback taskFailedCallback) { + super(threadFactory, tickDuration, unit, ticksPerWheel); + + this.executor = asyncExecutor; this.taskFailedCallback = taskFailedCallback; } @@ -85,14 +106,14 @@ public AsyncTimerTask(TimerTask timerTask) { /** */ @Override - public void run(Timeout timeout) throws Exception { + public void run(Timeout timeout) { this.timeout = timeout; try { AsyncHashedWheelTimer.this.executor.execute(this); } catch (RejectedExecutionException e) { taskFailedCallback.executionRejected(e); } catch (Throwable e) { - taskFailedCallback.executionRejected(e); + taskFailedCallback.executionFailed(e); } } diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/timer/RecycleAsyncHashedWheelTimer.java b/server/common/util/src/main/java/com/alipay/sofa/registry/timer/RecycleAsyncHashedWheelTimer.java new file mode 100644 index 000000000..fb3a0bd63 --- /dev/null +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/timer/RecycleAsyncHashedWheelTimer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.timer; + +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; + +import io.netty.util.Timeout; +import io.netty.util.TimerTask; + +/** + * + * @author kezhu.wukz + * @version $Id: RecycleAsyncHashedWheelTimer.java, v 0.1 2019-07-27 10:54 kezhu.wukz Exp $ + */ +public class RecycleAsyncHashedWheelTimer extends AsyncHashedWheelTimer { + + public RecycleAsyncHashedWheelTimer(ThreadFactory threadFactory, long tickDuration, + TimeUnit unit, int ticksPerWheel, int threadSize, + int queueSize, ThreadFactory asyncThreadFactory, + TaskFailedCallback taskFailedCallback) { + super(threadFactory, tickDuration, unit, ticksPerWheel, threadSize, queueSize, + asyncThreadFactory, taskFailedCallback); + } + + public RecycleAsyncHashedWheelTimer(ThreadFactory threadFactory, long tickDuration, + TimeUnit unit, int ticksPerWheel, Executor asyncExecutor, + TaskFailedCallback taskFailedCallback) { + super(threadFactory, tickDuration, unit, ticksPerWheel, asyncExecutor, taskFailedCallback); + } + + public Timeout newTimeout(TimerTask task, long firstDelay, long recycleDelay, TimeUnit unit, + BooleanSupplier checkCondition) { + + return super.newTimeout( + new RecycleAsyncTimerTask(task, recycleDelay, unit, checkCondition), firstDelay, unit); + } + + public Timeout newTimeout(TimerTask task, long recycleDelay, TimeUnit unit, + BooleanSupplier checkCondition) { + + return super + .newTimeout(new RecycleAsyncTimerTask(task, recycleDelay, unit, checkCondition), + recycleDelay, unit); + } + + /** + * + */ + class RecycleAsyncTimerTask implements TimerTask, Runnable { + /** */ + private final TimerTask timerTask; + + private final long recycleDelay; + + private final TimeUnit delayUnit; + + private final BooleanSupplier checkCondition; + + /** */ + private Timeout timeout; + + /** + * @param timerTask + */ + public RecycleAsyncTimerTask(TimerTask timerTask, long recycleDelay, TimeUnit unit, + BooleanSupplier checkCondition) { + super(); + this.timerTask = timerTask; + this.recycleDelay = recycleDelay; + this.delayUnit = unit; + this.checkCondition = checkCondition; + } + + /** + */ + @Override + public void run(Timeout timeout) { + this.timeout = timeout; + try { + RecycleAsyncHashedWheelTimer.super.executor.execute(this); + } catch (RejectedExecutionException e) { + taskFailedCallback.executionRejected(e); + } catch (Throwable e) { + taskFailedCallback.executionFailed(e); + } + } + + /** + * @see java.lang.Runnable#run() + */ + @Override + public void run() { + try { + this.timerTask.run(this.timeout); + } catch (Throwable e) { + taskFailedCallback.executionFailed(e); + } finally { + if (checkCondition.getAsBoolean()) { + RecycleAsyncHashedWheelTimer.this.newTimeout(timerTask, recycleDelay, + delayUnit, checkCondition); + } + } + } + + } + +} \ No newline at end of file diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/util/DatumVersionUtil.java b/server/common/util/src/main/java/com/alipay/sofa/registry/util/DatumVersionUtil.java new file mode 100644 index 000000000..78112433b --- /dev/null +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/util/DatumVersionUtil.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.util; + +/** + * generates ID: 49 bit millisecond timestamp + 15 bit incremental ID + * + * refer to: https://github.com/twitter/snowflake + * + * @author kezhu.wukz + * @version $Id: DatumVersionUtil.java, v 0.1 2019-07-04 22:05 kezhu.wukz Exp $ + */ +public class DatumVersionUtil { + + private static long sequence = 0L; + + /** Tue Jan 01 00:00:00 CST 2019 */ + private static long twepoch = 1546272000000L; + + private static long sequenceBits = 15L; + private static long timestampLeftShift = sequenceBits; + private static long sequenceMask = -1L ^ (-1L << sequenceBits); + + private static long lastTimestamp = -1L; + + public synchronized static long nextId() { + long timestamp = timeGen(); + + if (timestamp < lastTimestamp) { + throw new RuntimeException(String.format( + "Clock moved backwards. Refusing to generate id for %d milliseconds", lastTimestamp + - timestamp)); + } + + if (lastTimestamp == timestamp) { + sequence = (sequence + 1) & sequenceMask; + if (sequence == 0) { + timestamp = untilNextMillis(lastTimestamp); + } + } else { + sequence = 0L; + } + + lastTimestamp = timestamp; + + return ((timestamp - twepoch) << timestampLeftShift) | sequence; + } + + public static long getRealTimestamp(long id) { + return (id >> timestampLeftShift) + twepoch; + } + + private static long untilNextMillis(long lastTimestamp) { + long timestamp = timeGen(); + while (timestamp <= lastTimestamp) { + timestamp = timeGen(); + } + return timestamp; + } + + private static long timeGen() { + return System.currentTimeMillis(); + } + +} \ No newline at end of file diff --git a/server/common/util/src/test/java/com/alipay/sofa/registry/task/listener/TaskListenerTest.java b/server/common/util/src/test/java/com/alipay/sofa/registry/task/listener/TaskListenerTest.java index 9443f5d44..1e7d0acd5 100644 --- a/server/common/util/src/test/java/com/alipay/sofa/registry/task/listener/TaskListenerTest.java +++ b/server/common/util/src/test/java/com/alipay/sofa/registry/task/listener/TaskListenerTest.java @@ -16,13 +16,15 @@ */ package com.alipay.sofa.registry.task.listener; +import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.CANCEL_DATA_TASK; +import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.DATA_PUSH_TASK; +import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.WATCHER_REGISTER_FETCH_TASK; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.CANCEL_DATA_TASK; -import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.DATA_PUSH_TASK; -import static com.alipay.sofa.registry.task.listener.TaskEvent.TaskType.WATCHER_REGISTER_FETCH_TASK; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; /** * @author xuanbei @@ -65,8 +67,8 @@ public void doTest() { private static class DataPushTaskListener implements TaskListener { @Override - public boolean support(TaskEvent event) { - return DATA_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return DATA_PUSH_TASK; } @Override @@ -77,8 +79,8 @@ public void handleEvent(TaskEvent event) { private static class CancelDataTaskListener implements TaskListener { @Override - public boolean support(TaskEvent event) { - return TaskEvent.TaskType.CANCEL_DATA_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.CANCEL_DATA_TASK; } @Override @@ -89,8 +91,8 @@ public void handleEvent(TaskEvent event) { private static class WatcherRegisterFetchTaskListener implements TaskListener { @Override - public boolean support(TaskEvent event) { - return TaskEvent.TaskType.WATCHER_REGISTER_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.WATCHER_REGISTER_FETCH_TASK; } @Override diff --git a/server/common/util/src/test/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimerTest.java b/server/common/util/src/test/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimerTest.java index 11b63f53b..dbabd7faf 100644 --- a/server/common/util/src/test/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimerTest.java +++ b/server/common/util/src/test/java/com/alipay/sofa/registry/timer/AsyncHashedWheelTimerTest.java @@ -16,13 +16,15 @@ */ package com.alipay.sofa.registry.timer; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.netty.util.Timer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + import org.junit.Assert; import org.junit.Test; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import io.netty.util.Timer; /** * @author xuanbei @@ -40,7 +42,7 @@ public void doTest() throws InterruptedException { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setDaemon(true); final Timer timer = new AsyncHashedWheelTimer(threadFactoryBuilder.setNameFormat( - "AsyncHashedWheelTimerTest").build(), 50, TimeUnit.MILLISECONDS, 10, + "AsyncHashedWheelTimerTest").build(), 50, TimeUnit.MILLISECONDS, 10, 5, 10, threadFactoryBuilder.setNameFormat("Registry-DataNodeServiceImpl-WheelExecutor-%d") .build(), new AsyncHashedWheelTimer.TaskFailedCallback() { @Override diff --git a/server/consistency/pom.xml b/server/consistency/pom.xml index 7e8855042..af9efc807 100644 --- a/server/consistency/pom.xml +++ b/server/consistency/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/consistency/src/main/java/com/alipay/sofa/registry/consistency/hash/ConsistentHash.java b/server/consistency/src/main/java/com/alipay/sofa/registry/consistency/hash/ConsistentHash.java index 3c6949236..fdcbd2ad6 100644 --- a/server/consistency/src/main/java/com/alipay/sofa/registry/consistency/hash/ConsistentHash.java +++ b/server/consistency/src/main/java/com/alipay/sofa/registry/consistency/hash/ConsistentHash.java @@ -148,4 +148,59 @@ public List getNUniqueNodesFor(Object key, int n) { } return list; } + + /** + * This returns the closest n unique nodes in order for the object. + * + * This will return a list that has all nodes if n > number of nodes. + * + * @param key the key + * @param n the n + * @param disasterList disaster region name + * @return the n unique nodes for + */ + public List getNUniqueNodesFor(Object key, int n, List disasterList) { + if (circle.isEmpty()) { + return Collections.emptyList(); + } + + if (n > realNodes.size()) { + n = realNodes.size(); + } + + List disasters = disasterList != null && !disasterList.isEmpty() ? disasterList + : new ArrayList<>(); + List list = new ArrayList<>(n); + int hash = hashFunction.hash(key); + for (int i = 0; i < n; i++) { + if (!circle.containsKey(hash)) { + // go to next element. + SortedMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey(); + } + T candidate = circle.get(hash); + if (!list.contains(candidate)) { + + while (!disasters.isEmpty() && !disasters.contains(candidate.getNodeName())) { + hash++; + if (!circle.containsKey(hash)) { + // go to next element. + SortedMap tailMap = circle.tailMap(hash); + hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey(); + } + candidate = circle.get(hash); + } + list.add(candidate); + if (!disasters.isEmpty()) { + disasters.remove(candidate.getNodeName()); + } + + } else { + i--; // try again. + } + // find the next element in the circle + hash++; + } + return list; + } } diff --git a/server/distribution/data/conf/application.properties b/server/distribution/data/conf/application.properties index f5ed68c49..28027cb25 100644 --- a/server/distribution/data/conf/application.properties +++ b/server/distribution/data/conf/application.properties @@ -1,3 +1,2 @@ nodes.metaNode=DefaultDataCenter:localhost -nodes.localDataCenter=DefaultDataCenter -data.server.numberOfReplicas=1 \ No newline at end of file +nodes.localDataCenter=DefaultDataCenter \ No newline at end of file diff --git a/server/distribution/data/distribution-data.xml b/server/distribution/data/distribution-data.xml index b6a029b0e..630e83144 100644 --- a/server/distribution/data/distribution-data.xml +++ b/server/distribution/data/distribution-data.xml @@ -22,17 +22,17 @@ ../../server/data/target/registry-server-data-executable.jar - / + ./ registry-data.jar ../../server/data/src/main/resources/logback-spring.xml - /conf + ./conf logback-spring.xml ../version - / + ./ version_${project.version} diff --git a/server/distribution/data/pom.xml b/server/distribution/data/pom.xml index 12bbdfe43..00219e935 100644 --- a/server/distribution/data/pom.xml +++ b/server/distribution/data/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/distribution/integration/conf/application.properties.example b/server/distribution/integration/conf/application.properties.example index 3ecb93291..513b6c425 100644 --- a/server/distribution/integration/conf/application.properties.example +++ b/server/distribution/integration/conf/application.properties.example @@ -42,9 +42,9 @@ session.server.printTask.fixedDelay=30000 session.server.schedulerCheckVersionTimeout=3 session.server.schedulerCheckVersionFirstDelay=3 session.server.schedulerCheckVersionExpBackOffBound=10 -session.server.schedulerHeartbeatTimeout=30 -session.server.schedulerHeartbeatFirstDelay=30 -session.server.schedulerHeartbeatExpBackOffBound=10 +session.server.schedulerHeartbeatTimeout=3 +session.server.schedulerHeartbeatFirstDelay=3 +session.server.schedulerHeartbeatExpBackOffBound=1 session.server.schedulerFetchDataTimeout=3 session.server.schedulerFetchDataFirstDelay=3 session.server.schedulerFetchDataExpBackOffBound=10 diff --git a/server/distribution/integration/distribution-integration.xml b/server/distribution/integration/distribution-integration.xml index effdaf0fd..305504f53 100644 --- a/server/distribution/integration/distribution-integration.xml +++ b/server/distribution/integration/distribution-integration.xml @@ -22,17 +22,17 @@ ../../server/integration/target/registry-server-integration-executable.jar - / + ./ registry-integration.jar ../../server/integration/src/main/resources/logback-spring.xml - /conf + ./conf logback-spring.xml ../version - / + ./ version_${project.version} diff --git a/server/distribution/integration/pom.xml b/server/distribution/integration/pom.xml index d1bb77c6b..d316c6418 100644 --- a/server/distribution/integration/pom.xml +++ b/server/distribution/integration/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/distribution/meta/distribution-meta.xml b/server/distribution/meta/distribution-meta.xml index 29cf277c4..16dda1008 100644 --- a/server/distribution/meta/distribution-meta.xml +++ b/server/distribution/meta/distribution-meta.xml @@ -22,17 +22,17 @@ ../../server/meta/target/registry-server-meta-executable.jar - / + ./ registry-meta.jar ../../server/meta/src/main/resources/logback-spring.xml - /conf + ./conf logback-spring.xml ../version - / + ./ version_${project.version} diff --git a/server/distribution/meta/pom.xml b/server/distribution/meta/pom.xml index 3d7303499..de54ed413 100644 --- a/server/distribution/meta/pom.xml +++ b/server/distribution/meta/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/distribution/pom.xml b/server/distribution/pom.xml index 2e13d59b6..03eb6676e 100644 --- a/server/distribution/pom.xml +++ b/server/distribution/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/distribution/session/conf/application.properties.example b/server/distribution/session/conf/application.properties.example index 76a5e17f2..80a9f918a 100644 --- a/server/distribution/session/conf/application.properties.example +++ b/server/distribution/session/conf/application.properties.example @@ -19,9 +19,9 @@ session.server.printTask.fixedDelay=30000 session.server.schedulerCheckVersionTimeout=3 session.server.schedulerCheckVersionFirstDelay=3 session.server.schedulerCheckVersionExpBackOffBound=10 -session.server.schedulerHeartbeatTimeout=30 -session.server.schedulerHeartbeatFirstDelay=30 -session.server.schedulerHeartbeatExpBackOffBound=10 +session.server.schedulerHeartbeatTimeout=3 +session.server.schedulerHeartbeatFirstDelay=3 +session.server.schedulerHeartbeatExpBackOffBound=1 session.server.schedulerFetchDataTimeout=3 session.server.schedulerFetchDataFirstDelay=3 session.server.schedulerFetchDataExpBackOffBound=10 diff --git a/server/distribution/session/distribution-session.xml b/server/distribution/session/distribution-session.xml index 81849467b..06175cb8d 100644 --- a/server/distribution/session/distribution-session.xml +++ b/server/distribution/session/distribution-session.xml @@ -22,17 +22,17 @@ ../../server/session/target/registry-server-session-executable.jar - / + ./ registry-session.jar ../../server/session/src/main/resources/logback-spring.xml - /conf + ./conf logback-spring.xml ../version - / + ./ version_${project.version} diff --git a/server/distribution/session/pom.xml b/server/distribution/session/pom.xml index e7061c240..750e37529 100644 --- a/server/distribution/session/pom.xml +++ b/server/distribution/session/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/pom.xml b/server/pom.xml index 23abe7efe..cdc71eb9f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -7,7 +7,7 @@ com.alipay.sofa registry-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml diff --git a/server/remoting/api/pom.xml b/server/remoting/api/pom.xml index 4be3abfe7..fcb259197 100644 --- a/server/remoting/api/pom.xml +++ b/server/remoting/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/CallbackHandler.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/CallbackHandler.java index 2b32e72a0..aa73eeb46 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/CallbackHandler.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/CallbackHandler.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.remoting; +import java.util.concurrent.Executor; + /** * * @author shangyu.wh @@ -38,4 +40,10 @@ public interface CallbackHandler { * @param exception */ void onException(Channel channel, Throwable exception); + + /** + * override executor + * @return + */ + Executor getExecutor(); } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Channel.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Channel.java index 4f51ffed2..162c59ca2 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Channel.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Channel.java @@ -16,9 +16,10 @@ */ package com.alipay.sofa.registry.remoting; -import javax.ws.rs.client.WebTarget; import java.net.InetSocketAddress; +import javax.ws.rs.client.WebTarget; + /** * * @author shangyu.wh @@ -68,4 +69,6 @@ public interface Channel { * @return */ WebTarget getWebTarget(); + + void close(); } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Client.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Client.java index f19aba5a6..aa08d9c3e 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Client.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Client.java @@ -18,9 +18,6 @@ import com.alipay.sofa.registry.common.model.store.URL; -import java.net.InetSocketAddress; -import java.util.Collection; - /** * * @author shangyu.wh @@ -28,21 +25,6 @@ */ public interface Client extends Endpoint { - /** - * get channels. - * - * @return channels - */ - Collection getChannels(); - - /** - * get channel. - * - * @param remoteAddress - * @return channel - */ - Channel getChannel(InetSocketAddress remoteAddress); - /** * get channel by url. * @@ -57,4 +39,35 @@ public interface Client extends Endpoint { * @return */ Channel connect(URL url); + + /** + * Sync send + * + * @param url the url + * @param message the message + * @param timeoutMillis the timeout millis + * @return object + */ + Object sendSync(final URL url, final Object message, final int timeoutMillis); + + /** + * Sync send + * + * @param channel the channel + * @param message the message + * @param timeoutMillis the timeout millis + * @return object + */ + Object sendSync(final Channel channel, final Object message, final int timeoutMillis); + + /** + * send with callback handler + * + * @param url the url + * @param message the message + * @param callbackHandler the callback handler + * @param timeoutMillis the timeout millis + */ + void sendCallback(final URL url, final Object message, CallbackHandler callbackHandler, + final int timeoutMillis); } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Endpoint.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Endpoint.java index 10cff4f22..d1b23ab5f 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Endpoint.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Endpoint.java @@ -17,22 +17,16 @@ package com.alipay.sofa.registry.remoting; import java.net.InetSocketAddress; -import java.util.List; /** * The interface Endpoint. + * * @author shangyu.wh + * @author kezhu.wukz * @version $Id : Endpoint.java, v 0.1 2017-11-20 20:03 shangyu.wh Exp $ */ public interface Endpoint { - /** - * get channel handlers. - * - * @return channel handlers - */ - List getChannelHandlers(); - /** * get local address. * @@ -45,13 +39,6 @@ public interface Endpoint { */ void close(); - /** - * Close. - * - * @param channel the channel - */ - void close(final Channel channel); - /** * is closed. * @@ -59,31 +46,4 @@ public interface Endpoint { */ boolean isClosed(); - /** - * oneway send - * @param channel the channel - * @param message the message - */ - void sendOneway(final Channel channel, final Object message); - - /** - * Sync send - * - * @param channel the channel - * @param message the message - * @param timeoutMillis the timeout millis - * @return object - */ - Object sendSync(final Channel channel, final Object message, final int timeoutMillis); - - /** - * send with callback handler - * - * @param channel the channel - * @param message the message - * @param callbackHandler the callback handler - * @param timeoutMillis the timeout millis - */ - void sendCallback(final Channel channel, final Object message, CallbackHandler callbackHandler, - final int timeoutMillis); } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Server.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Server.java index 7452bb4f4..5b3de39d7 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Server.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/Server.java @@ -16,11 +16,11 @@ */ package com.alipay.sofa.registry.remoting; -import com.alipay.sofa.registry.common.model.store.URL; - import java.net.InetSocketAddress; import java.util.Collection; +import com.alipay.sofa.registry.common.model.store.URL; + /** * * @author shangyu.wh @@ -57,4 +57,34 @@ public interface Server extends Endpoint { * @return channel */ Channel getChannel(URL url); + + /** + * close the channel. + * + * @param channel + */ + void close(Channel channel); + + int getChannelCount(); + + /** + * Sync send + * + * @param channel the channel + * @param message the message + * @param timeoutMillis the timeout millis + * @return object + */ + Object sendSync(final Channel channel, final Object message, final int timeoutMillis); + + /** + * send with callback handler + * + * @param channel the channel + * @param message the message + * @param callbackHandler the callback handler + * @param timeoutMillis the timeout millis + */ + void sendCallback(final Channel channel, final Object message, CallbackHandler callbackHandler, + final int timeoutMillis); } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/Exchange.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/Exchange.java index 961167107..f0f286413 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/Exchange.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/Exchange.java @@ -41,6 +41,18 @@ public interface Exchange { */ Client connect(String serverType, URL serverUrl, T... channelHandlers); + /** + * connect same type server,one server ip one connection + * such as different server on data server,serverOne and serverTwo,different type server must match different channelHandlers, + * so we must connect by serverType,and get Client instance by serverType + * @param serverType + * @param connNum connection number per serverUrl + * @param serverUrl + * @param channelHandlers + * @return + */ + Client connect(String serverType, int connNum, URL serverUrl, T... channelHandlers); + /** * bind server by server port in url parameter,one port must by same server type * @param url diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/RequestException.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/RequestException.java index 3340bbf4b..1d5fa3e06 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/RequestException.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/RequestException.java @@ -25,7 +25,8 @@ */ public class RequestException extends Exception { - private Request request; + private static final int MAX_BODY_SIZE = 512; + private Request request; /** * constructor @@ -77,14 +78,19 @@ public RequestException(Throwable cause) { * get requestInfo from Request * @return */ - public String getRequestMessage() { + @Override + public String getMessage() { StringBuilder sb = new StringBuilder(); if (request != null) { - sb.append("Request url:").append(request.getRequestUrl()).append(" body:") - .append(request.getRequestBody()); - } else { - sb.append("Request data can not be null!"); + String requestBody = request.getRequestBody() != null ? request.getRequestBody() + .toString() : "null"; + if (requestBody.length() > MAX_BODY_SIZE) { + requestBody = requestBody.substring(0, MAX_BODY_SIZE); + } + sb.append("request url: ").append(request.getRequestUrl()).append(", body: ") + .append(requestBody).append(", "); } + sb.append(super.getMessage()); return sb.toString(); } } \ No newline at end of file diff --git a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/message/Request.java b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/message/Request.java index 7bf15fff0..473ed2fc4 100644 --- a/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/message/Request.java +++ b/server/remoting/api/src/main/java/com/alipay/sofa/registry/remoting/exchange/message/Request.java @@ -16,11 +16,11 @@ */ package com.alipay.sofa.registry.remoting.exchange.message; +import java.util.concurrent.atomic.AtomicInteger; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.remoting.CallbackHandler; -import java.util.concurrent.atomic.AtomicInteger; - /** * The interface Request. * @@ -60,4 +60,8 @@ default CallbackHandler getCallBackHandler() { default AtomicInteger getRetryTimes() { return new AtomicInteger(); } + + default Integer getTimeout() { + return null; + } } \ No newline at end of file diff --git a/server/remoting/api/src/test/java/com/alipay/sofa/registry/remoting/RequestExceptionTest.java b/server/remoting/api/src/test/java/com/alipay/sofa/registry/remoting/RequestExceptionTest.java index 7f98cc690..03c7f8526 100644 --- a/server/remoting/api/src/test/java/com/alipay/sofa/registry/remoting/RequestExceptionTest.java +++ b/server/remoting/api/src/test/java/com/alipay/sofa/registry/remoting/RequestExceptionTest.java @@ -16,11 +16,12 @@ */ package com.alipay.sofa.registry.remoting; +import org.junit.Assert; +import org.junit.Test; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.remoting.exchange.RequestException; import com.alipay.sofa.registry.remoting.exchange.message.Request; -import org.junit.Assert; -import org.junit.Test; /** * @author xuanbei @@ -31,18 +32,16 @@ public class RequestExceptionTest { public void doTest() { RequestException exception = new RequestException("error message"); Assert.assertEquals("error message", exception.getMessage()); - Assert.assertEquals("Request data can not be null!", exception.getRequestMessage()); RuntimeException runtimeException = new RuntimeException("error message"); exception = new RequestException(runtimeException); Assert.assertEquals("java.lang.RuntimeException: error message", exception.getMessage()); Assert.assertEquals(runtimeException, exception.getCause()); - Assert.assertEquals("Request data can not be null!", exception.getRequestMessage()); + Assert.assertEquals("java.lang.RuntimeException: error message", exception.getMessage()); exception = new RequestException("error message", runtimeException); Assert.assertEquals("error message", exception.getMessage()); Assert.assertEquals(runtimeException, exception.getCause()); - Assert.assertEquals("Request data can not be null!", exception.getRequestMessage()); Request request = new Request() { @Override @@ -56,11 +55,11 @@ public URL getRequestUrl() { } }; exception = new RequestException("error message", request); - Assert.assertEquals("error message", exception.getMessage()); - Assert.assertEquals("Request url:null body:request body", exception.getRequestMessage()); + Assert.assertEquals("request url: null, body: request body, error message", + exception.getMessage()); exception = new RequestException("error message", request, runtimeException); - Assert.assertEquals("error message", exception.getMessage()); - Assert.assertEquals("Request url:null body:request body", exception.getRequestMessage()); + Assert.assertEquals("request url: null, body: request body, error message", + exception.getMessage()); } } diff --git a/server/remoting/bolt/pom.xml b/server/remoting/bolt/pom.xml index 21262538c..3c256443a 100644 --- a/server/remoting/bolt/pom.xml +++ b/server/remoting/bolt/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltChannel.java b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltChannel.java index 0e857268d..9a4ddd334 100644 --- a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltChannel.java +++ b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltChannel.java @@ -16,19 +16,21 @@ */ package com.alipay.sofa.registry.remoting.bolt; +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.ws.rs.client.WebTarget; + import com.alipay.remoting.AsyncContext; import com.alipay.remoting.BizContext; import com.alipay.remoting.Connection; import com.alipay.sofa.registry.remoting.Channel; -import javax.ws.rs.client.WebTarget; -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - /** * * @author shangyu.wh + * @author kezhu.wukz * @version $Id: BoltChannel.java, v 0.1 2017-11-24 16:46 shangyu.wh Exp $ */ public class BoltChannel implements Channel { @@ -84,6 +86,11 @@ public WebTarget getWebTarget() { return null; } + @Override + public void close() { + this.connection.close(); + } + /** * Getter method for property connection. * diff --git a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltClient.java b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltClient.java index ad09d9b31..b7dc0b04d 100644 --- a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltClient.java +++ b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltClient.java @@ -16,11 +16,19 @@ */ package com.alipay.sofa.registry.remoting.bolt; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + import com.alipay.remoting.Connection; +import com.alipay.remoting.ConnectionEventProcessor; import com.alipay.remoting.ConnectionEventType; import com.alipay.remoting.InvokeCallback; +import com.alipay.remoting.Url; import com.alipay.remoting.exception.RemotingException; import com.alipay.remoting.rpc.RpcClient; +import com.alipay.remoting.rpc.protocol.RpcProtocol; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -32,151 +40,136 @@ import com.alipay.sofa.registry.remoting.ChannelHandler.InvokeType; import com.alipay.sofa.registry.remoting.Client; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - /** * The type Bolt client. + * + * @author kezhu.wukz * @author shangyu.wh * @version $Id : BoltClient.java, v 0.1 2017-11-27 14:46 shangyu.wh Exp $ */ public class BoltClient implements Client { - private static final Logger LOGGER = LoggerFactory.getLogger(BoltClient.class); - private RpcClient boltClient; - private List channelHandlers = new ArrayList<>(); - private Map channels = new HashMap<>(); - private AtomicBoolean initHandler = new AtomicBoolean(false); + private static final Logger LOGGER = LoggerFactory.getLogger(BoltClient.class); + + private RpcClient rpcClient; + + private AtomicBoolean closed = new AtomicBoolean(false); + + private int connectTimeout = 2000; + + private final int connNum; /** * Instantiates a new Bolt client. */ - public BoltClient() { - boltClient = new RpcClient(); - boltClient.init(); - } + public BoltClient(int connNum) { + rpcClient = new RpcClient(); + rpcClient.init(); - @Override - public Channel connect(URL targetUrl) { + this.connNum = connNum; + } - if (targetUrl == null) { - throw new IllegalArgumentException("Create connection targetUrl can not be null!"); - } - InetSocketAddress address = URL.toInetSocketAddress(targetUrl); - Channel c = getChannel(address); - if (c != null && c.isConnected()) { - LOGGER.info("Target url:" + targetUrl + " has been connected!", targetUrl); - return c; + /** + * Setter method for property channelHandlers. + * + * @param channelHandlers value to be assigned to property channelHandlers + */ + public void initHandlers(List channelHandlers) { + ChannelHandler connectionEventHandler = null; + for (ChannelHandler channelHandler : channelHandlers) { + if (HandlerType.LISENTER.equals(channelHandler.getType())) { + connectionEventHandler = channelHandler; + break; + } } - initHandler(); - - try { - Connection connection = boltClient - .getConnection(NetUtil.toAddressString(address), 1000); - if (connection != null) { - BoltChannel channel = new BoltChannel(); - channel.setConnection(connection); - channels.put(connection.getUrl().getOriginUrl(), channel); - return channel; - } else { - throw new RuntimeException("Bolt client connect server get none connection!"); + rpcClient.addConnectionEventProcessor(ConnectionEventType.CONNECT, + newConnectionEventAdapter(connectionEventHandler, ConnectionEventType.CONNECT)); + rpcClient.addConnectionEventProcessor(ConnectionEventType.CLOSE, + newConnectionEventAdapter(connectionEventHandler, ConnectionEventType.CLOSE)); + rpcClient.addConnectionEventProcessor(ConnectionEventType.EXCEPTION, + newConnectionEventAdapter(connectionEventHandler, ConnectionEventType.EXCEPTION)); + + for (ChannelHandler channelHandler : channelHandlers) { + if (HandlerType.PROCESSER.equals(channelHandler.getType())) { + if (InvokeType.SYNC.equals(channelHandler.getInvokeType())) { + rpcClient.registerUserProcessor(newSyncProcessor(channelHandler)); + } else { + rpcClient.registerUserProcessor(newAsyncProcessor(channelHandler)); + } } - - } catch (RemotingException e) { - LOGGER.error("Bolt client connect server got a RemotingException! target url:" - + targetUrl, e); - throw new RuntimeException("Bolt client connect server got a RemotingException!", e); - } catch (InterruptedException e) { - LOGGER.error("Bolt client connect server has been Interrupted!", e); - throw new RuntimeException("Bolt client connect server has been Interrupted!", e); } } - private void initHandler() { - - if (initHandler.compareAndSet(false, true)) { - boltClient.addConnectionEventProcessor(ConnectionEventType.CONNECT, - new ConnectionEventAdapter(ConnectionEventType.CONNECT, - getConnectionEventHandler(), null)); - boltClient.addConnectionEventProcessor(ConnectionEventType.CLOSE, - new ConnectionEventAdapter(ConnectionEventType.CLOSE, getConnectionEventHandler(), - null)); - boltClient.addConnectionEventProcessor(ConnectionEventType.EXCEPTION, - new ConnectionEventAdapter(ConnectionEventType.EXCEPTION, - getConnectionEventHandler(), null)); + protected ConnectionEventProcessor newConnectionEventAdapter(ChannelHandler connectionEventHandler, + ConnectionEventType connectEventType) { + return new ConnectionEventAdapter(connectEventType, connectionEventHandler, null); + } - registerUserProcessorHandler(); - } + protected AsyncUserProcessorAdapter newAsyncProcessor(ChannelHandler channelHandler) { + return new AsyncUserProcessorAdapter(channelHandler); } - private void registerUserProcessorHandler() { - if (channelHandlers != null) { - for (ChannelHandler channelHandler : channelHandlers) { - if (HandlerType.PROCESSER.equals(channelHandler.getType())) { - if (InvokeType.SYNC.equals(channelHandler.getInvokeType())) { - boltClient.registerUserProcessor(new SyncUserProcessorAdapter( - channelHandler)); - } else { - boltClient.registerUserProcessor(new AsyncUserProcessorAdapter( - channelHandler)); - } - } - } - } + protected SyncUserProcessorAdapter newSyncProcessor(ChannelHandler channelHandler) { + return new SyncUserProcessorAdapter(channelHandler); } @Override - public Collection getChannels() { - - Collection chs = new HashSet<>(); - for (Channel channel : this.channels.values()) { - if (channel.isConnected()) { - chs.add(channel); - } else { - channels.remove(NetUtil.toAddressString(channel.getRemoteAddress())); - } + public Channel connect(URL url) { + if (url == null) { + throw new IllegalArgumentException("Create connection url can not be null!"); } - return chs; - } + try { + Connection connection = getBoltConnection(rpcClient, url); + BoltChannel channel = new BoltChannel(); + channel.setConnection(connection); + return channel; - @Override - public Channel getChannel(InetSocketAddress remoteAddress) { - Channel c = channels.get(NetUtil.toAddressString(remoteAddress)); - if (c == null || !c.isConnected()) { - return null; + } catch (RemotingException e) { + LOGGER + .error("Bolt client connect server got a RemotingException! target url:" + url, e); + throw new RuntimeException("Bolt client connect server got a RemotingException!", e); } - return c; } - @Override - public Channel getChannel(URL url) { - Channel c = channels.get(url.getAddressString()); - if (c == null || !c.isConnected()) { - return null; + protected Connection getBoltConnection(RpcClient rpcClient, URL url) throws RemotingException { + Url boltUrl = createBoltUrl(url); + try { + Connection connection = rpcClient.getConnection(boltUrl, connectTimeout); + if (connection == null || !connection.isFine()) { + if (connection != null) { + connection.close(); + } + throw new RemotingException("Get bolt connection failed for boltUrl: " + boltUrl); + } + return connection; + } catch (InterruptedException e) { + throw new RuntimeException( + "BoltClient rpcClient.getConnection InterruptedException! target boltUrl:" + + boltUrl, e); } - return c; } - @Override - public List getChannelHandlers() { - return channelHandlers; + protected Url createBoltUrl(URL url) { + Url boltUrl = new Url(url.getIpAddress(), url.getPort()); + boltUrl.setProtocol(RpcProtocol.PROTOCOL_CODE); + boltUrl.setConnNum(connNum); + boltUrl.setConnWarmup(true); + return boltUrl; } - /** - * Setter method for property channelHandlers. - * - * @param channelHandlers value to be assigned to property channelHandlers - */ - public void setChannelHandlers(List channelHandlers) { - this.channelHandlers = channelHandlers; + @Override + public Channel getChannel(URL url) { + try { + Connection connection = getBoltConnection(rpcClient, url); + BoltChannel channel = new BoltChannel(); + channel.setConnection(connection); + return channel; + } catch (RemotingException e) { + LOGGER + .error("Bolt client connect server got a RemotingException! target url:" + url, e); + throw new RuntimeException("Bolt client connect server got a RemotingException!", e); + } } @Override @@ -184,100 +177,46 @@ public InetSocketAddress getLocalAddress() { return NetUtil.getLocalSocketAddress(); } - /** - * Gets connection event handler. - * - * @return the connection event handler - */ - public ChannelHandler getConnectionEventHandler() { - if (channelHandlers != null) { - for (ChannelHandler channelHandler : channelHandlers) { - if (HandlerType.LISENTER.equals(channelHandler.getType())) { - return channelHandler; - } - } - } - return null; - } - @Override public void close() { - Collection chs = getChannels(); - if (chs != null && chs.size() > 0) { - for (Channel ch : chs) { - if (ch != null) { - boltClient.closeStandaloneConnection(((BoltChannel) ch).getConnection()); - } - } - } - } - - @Override - public void close(Channel channel) { - if (channel != null) { - Connection connection = ((BoltChannel) channel).getConnection(); - if (null != connection.getUrl() && null != connection.getUrl().getOriginUrl()) { - channels.remove(connection.getUrl().getOriginUrl()); - } - boltClient.closeStandaloneConnection(connection); + if (closed.compareAndSet(false, true)) { + rpcClient.shutdown(); } } @Override public boolean isClosed() { - boolean ret = false; - Collection chs = getChannels(); - if (chs != null && chs.size() > 0) { - for (Channel ch : chs) { - if (ch != null && !ch.isConnected()) { - ret = true; - break; - } - } - } else { - //has no channels - return true; - } - return ret; + return closed.get(); } @Override - public void sendOneway(Channel channel, Object message) { - if (channel != null && channel.isConnected()) { - if (channel instanceof BoltChannel) { - BoltChannel boltChannel = (BoltChannel) channel; - try { - boltClient.oneway(boltChannel.getConnection(), message); - } catch (RemotingException e) { - LOGGER.error("Bolt Client oneway request RemotingException! target url: {}", - boltChannel.getRemoteAddress(), e); - } - } + public Object sendSync(URL url, Object message, int timeoutMillis) { + try { + return rpcClient.invokeSync(createBoltUrl(url), message, timeoutMillis); + } catch (RemotingException e) { + String msg = "Bolt Client sendSync message RemotingException! target url:" + url; + LOGGER.error(msg, e); + throw new RuntimeException(msg, e); + } catch (InterruptedException e) { + throw new RuntimeException( + "Bolt Client sendSync message InterruptedException! target url:" + url, e); } } @Override public Object sendSync(Channel channel, Object message, int timeoutMillis) { - if (channel != null && channel.isConnected()) { - if (channel instanceof BoltChannel) { - BoltChannel boltChannel = (BoltChannel) channel; - try { - return boltClient.invokeSync(boltChannel.getConnection(), message, - timeoutMillis); - } catch (RemotingException e) { - LOGGER.error("Bolt Client sendSync message RemotingException! target url:" - + boltChannel.getRemoteAddress(), e); - throw new RuntimeException("Bolt Client sendSync message RemotingException!", e); - } catch (InterruptedException e) { - LOGGER.error("Bolt Client sendSync message InterruptedException! target url:" - + boltChannel.getRemoteAddress(), e); - throw new RuntimeException( - "Bolt Client sendSync message InterruptedException!", e); - } - } else { - throw new IllegalArgumentException("Input channel instance error! instance class:" - + channel.getClass().getName()); + BoltChannel boltChannel = (BoltChannel) channel; + try { + return rpcClient.invokeSync(boltChannel.getConnection(), message, timeoutMillis); + } catch (RemotingException e) { + LOGGER.error("Bolt Client sendSync message RemotingException! target boltUrl:" + + boltChannel.getRemoteAddress(), e); + throw new RuntimeException("Bolt Client sendSync message RemotingException!", e); + } catch (InterruptedException e) { + LOGGER.error("Bolt Client sendSync message InterruptedException! target boltUrl:" + + boltChannel.getRemoteAddress(), e); + throw new RuntimeException("Bolt Client sendSync message InterruptedException!", e); } } throw new IllegalArgumentException( @@ -286,44 +225,35 @@ public Object sendSync(Channel channel, Object message, int timeoutMillis) { } @Override - public void sendCallback(Channel channel, Object message, CallbackHandler callbackHandler, + public void sendCallback(URL url, Object message, CallbackHandler callbackHandler, int timeoutMillis) { - if (channel != null && channel.isConnected()) { - if (channel instanceof BoltChannel) { - BoltChannel boltChannel = (BoltChannel) channel; - try { - boltClient.invokeWithCallback(boltChannel.getConnection(), message, - new InvokeCallback() { - - @Override - public void onResponse(Object result) { - callbackHandler.onCallback(channel, result); - } + try { + Connection connection = getBoltConnection(rpcClient, url); + BoltChannel channel = new BoltChannel(); + channel.setConnection(connection); + rpcClient.invokeWithCallback(connection, message, new InvokeCallback() { + + @Override + public void onResponse(Object result) { + callbackHandler.onCallback(channel, result); + } - @Override - public void onException(Throwable e) { - callbackHandler.onException(channel, e); - } + @Override + public void onException(Throwable e) { + callbackHandler.onException(channel, e); + } - @Override - public Executor getExecutor() { - return null; - } - }, timeoutMillis); - return; - } catch (RemotingException e) { - LOGGER.error("Bolt Client sendSync message RemotingException! target url:" - + boltChannel.getRemoteAddress(), e); - throw new RuntimeException("Bolt Client sendSync message RemotingException!", e); + @Override + public Executor getExecutor() { + return callbackHandler.getExecutor(); } - } else { - throw new IllegalArgumentException("Input channel instance error! instance class:" - + channel.getClass().getName()); - } + }, timeoutMillis); + return; + } catch (RemotingException e) { + String msg = "Bolt Client sendSync message RemotingException! target url:" + url; + LOGGER.error(msg, e); + throw new RuntimeException(msg, e); } - throw new IllegalArgumentException( - "Input channel: " + channel - + " error! channel cannot be null,or channel must be connected!"); } } \ No newline at end of file diff --git a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltServer.java b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltServer.java index e9fc9ed38..547c073e8 100644 --- a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltServer.java +++ b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/BoltServer.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.remoting.bolt; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + import com.alipay.remoting.Connection; import com.alipay.remoting.ConnectionEventType; import com.alipay.remoting.InvokeCallback; @@ -33,16 +43,6 @@ import com.alipay.sofa.registry.remoting.ChannelHandler.InvokeType; import com.alipay.sofa.registry.remoting.Server; -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - /** * * @author shangyu.wh @@ -206,11 +206,6 @@ public Channel getChannel(URL url) { return null; } - @Override - public List getChannelHandlers() { - return channelHandlers; - } - @Override public InetSocketAddress getLocalAddress() { return new InetSocketAddress(url.getPort()); @@ -240,49 +235,27 @@ public boolean isClosed() { return !isStarted.get(); } - @Override - public void sendOneway(Channel channel, Object message) { - if (channel != null && channel.isConnected()) { - try { - Url url = new Url(channel.getRemoteAddress().getAddress().getHostAddress(), channel - .getRemoteAddress().getPort()); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Bolt Server one way message:{} , target url:{}", message, url); - } - boltServer.oneway(url, message); - } catch (RemotingException e) { - LOGGER.error("Bolt Server one way message RemotingException! target url:" + url, e); - throw new RuntimeException("Bolt Server one way message RemotingException!", e); - } catch (InterruptedException e) { - LOGGER.error("Bolt Server one way message InterruptedException! target url:" + url, - e); - throw new RuntimeException("Bolt Server one way message InterruptedException!", e); - } - } - throw new IllegalArgumentException( - "Send message connection can not be null or connection not be connected!"); - } - @Override public Object sendSync(Channel channel, Object message, int timeoutMillis) { if (channel != null && channel.isConnected()) { + Url boltUrl = null; try { - Url url = new Url(channel.getRemoteAddress().getAddress().getHostAddress(), channel + boltUrl = new Url(channel.getRemoteAddress().getAddress().getHostAddress(), channel .getRemoteAddress().getPort()); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Bolt Server sendSync message:{} , target url:{}", message, url); + LOGGER.debug("Bolt Server sendSync message:{} , target url:{}", message, + boltUrl); } - return boltServer.invokeSync(url, message, timeoutMillis); + return boltServer.invokeSync(boltUrl, message, timeoutMillis); } catch (RemotingException e) { - LOGGER - .error("Bolt Server sendSync message RemotingException! target url:" + url, e); + LOGGER.error("Bolt Server sendSync message RemotingException! target url:" + + boltUrl, e); throw new RuntimeException("Bolt Server sendSync message RemotingException!", e); } catch (InterruptedException e) { - LOGGER.error( - "Bolt Server sendSync message InterruptedException! target url:" + url, e); + LOGGER.error("Bolt Server sendSync message InterruptedException! target url:" + + boltUrl, e); throw new RuntimeException("Bolt Server sendSync message InterruptedException!", e); } } @@ -294,14 +267,16 @@ public Object sendSync(Channel channel, Object message, int timeoutMillis) { public void sendCallback(Channel channel, Object message, CallbackHandler callbackHandler, int timeoutMillis) { if (channel != null && channel.isConnected()) { + Url boltUrl = null; try { - Url url = new Url(channel.getRemoteAddress().getAddress().getHostAddress(), channel + boltUrl = new Url(channel.getRemoteAddress().getAddress().getHostAddress(), channel .getRemoteAddress().getPort()); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Bolt Server sendSync message:{} , target url:{}", message, url); + LOGGER.debug("Bolt Server sendSync message:{} , target url:{}", message, + boltUrl); } - boltServer.invokeWithCallback(url, message, new InvokeCallback() { + boltServer.invokeWithCallback(boltUrl, message, new InvokeCallback() { @Override public void onResponse(Object result) { callbackHandler.onCallback(channel, result); @@ -314,7 +289,7 @@ public void onException(Throwable e) { @Override public Executor getExecutor() { - return null; + return callbackHandler.getExecutor(); } }, timeoutMillis); return; @@ -322,7 +297,8 @@ public Executor getExecutor() { throw new RuntimeException("Bolt Server invoke with callback RemotingException!", e); } catch (InterruptedException e) { PUSH_LOGGER.error( - "Bolt Server invoke with callback InterruptedException! target url:" + url, e); + "Bolt Server invoke with callback InterruptedException! target url:" + boltUrl, + e); throw new RuntimeException( "Bolt Server invoke with callback InterruptedException!", e); } @@ -342,4 +318,9 @@ public void removeChannel(Channel channel) { public RpcServer getRpcServer() { return boltServer; } + + @Override + public int getChannelCount() { + return channels.size(); + } } \ No newline at end of file diff --git a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/exchange/BoltExchange.java b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/exchange/BoltExchange.java index 701729786..52627f664 100644 --- a/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/exchange/BoltExchange.java +++ b/server/remoting/bolt/src/main/java/com/alipay/sofa/registry/remoting/bolt/exchange/BoltExchange.java @@ -16,8 +16,11 @@ */ package com.alipay.sofa.registry.remoting.bolt.exchange; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import com.alipay.sofa.registry.common.model.store.URL; -import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.Client; import com.alipay.sofa.registry.remoting.Server; @@ -25,10 +28,6 @@ import com.alipay.sofa.registry.remoting.bolt.BoltServer; import com.alipay.sofa.registry.remoting.exchange.Exchange; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - /** * * @author shangyu.wh @@ -42,38 +41,26 @@ public class BoltExchange implements Exchange { @Override public Client connect(String serverType, URL serverUrl, ChannelHandler... channelHandlers) { + return this.connect(serverType, 1, serverUrl, channelHandlers); + } + @Override + public Client connect(String serverType, int connNum, URL serverUrl, ChannelHandler... channelHandlers) { if (channelHandlers == null) { throw new IllegalArgumentException("channelHandlers cannot be null!"); } - Client client = clients.get(serverType); - if (client == null) { - BoltClient boltClient = new BoltClient(); - boltClient.setChannelHandlers(Arrays.asList(channelHandlers)); - boltClient.connect(serverUrl); - client = clients.putIfAbsent(serverType, boltClient); - if (client == null) { - client = boltClient; - } - } else { - Channel channel = client.getChannel(serverUrl); - if (channel == null) { - BoltClient boltClient = (BoltClient) client; - boltClient.setChannelHandlers(Arrays.asList(channelHandlers)); - boltClient.connect(serverUrl); - } - } + Client client = clients.computeIfAbsent(serverType, key -> newBoltClient(connNum, channelHandlers)); + client.connect(serverUrl); return client; } @Override public Server open(URL url, ChannelHandler... channelHandlers) { - if (channelHandlers == null) { throw new IllegalArgumentException("channelHandlers cannot be null!"); } - BoltServer server = new BoltServer(url, Arrays.asList(channelHandlers)); + BoltServer server = createBoltServer(url, channelHandlers); setServer(server, url); server.startServer(); return server; @@ -97,4 +84,18 @@ public Server getServer(Integer port) { public void setServer(Server server, URL url) { serverMap.putIfAbsent(url.getPort(), server); } + + private BoltClient newBoltClient(int connNum, ChannelHandler[] channelHandlers) { + BoltClient boltClient = createBoltClient(connNum); + boltClient.initHandlers(Arrays.asList(channelHandlers)); + return boltClient; + } + + protected BoltClient createBoltClient(int connNum) { + return new BoltClient(connNum); + } + + protected BoltServer createBoltServer(URL url, ChannelHandler[] channelHandlers) { + return new BoltServer(url, Arrays.asList(channelHandlers)); + } } \ No newline at end of file diff --git a/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/BoltServerTest.java b/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/BoltServerTest.java index 36e8e3830..cfb25f7c8 100644 --- a/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/BoltServerTest.java +++ b/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/BoltServerTest.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.remoting.bolt; -import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.remoting.bolt.BoltServer; +import java.net.InetSocketAddress; + +import javax.ws.rs.client.WebTarget; + import org.junit.Assert; import org.junit.Test; -import javax.ws.rs.client.WebTarget; -import java.net.InetSocketAddress; +import com.alipay.sofa.registry.remoting.Channel; /** * @@ -67,6 +68,11 @@ public void setAttribute(String key, Object value) { public WebTarget getWebTarget() { return null; } + + @Override + public void close() { + + } }); } sessionServer.getChannels(); diff --git a/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/MockChannel.java b/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/MockChannel.java index b36f74838..d00c5d5b9 100644 --- a/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/MockChannel.java +++ b/server/remoting/bolt/src/test/java/com/alipay/sofa/registry/remoting/bolt/MockChannel.java @@ -16,10 +16,11 @@ */ package com.alipay.sofa.registry.remoting.bolt; -import com.alipay.sofa.registry.remoting.Channel; +import java.net.InetSocketAddress; import javax.ws.rs.client.WebTarget; -import java.net.InetSocketAddress; + +import com.alipay.sofa.registry.remoting.Channel; /** * @author xuanbei @@ -55,4 +56,9 @@ public void setAttribute(String key, Object value) { public WebTarget getWebTarget() { return null; } + + @Override + public void close() { + + } } diff --git a/server/remoting/http/pom.xml b/server/remoting/http/pom.xml index df9645432..c6f568351 100644 --- a/server/remoting/http/pom.xml +++ b/server/remoting/http/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyChannel.java b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyChannel.java index 54154d40f..5a061273a 100644 --- a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyChannel.java +++ b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyChannel.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.remoting.jersey; -import com.alipay.sofa.registry.net.NetUtil; -import com.alipay.sofa.registry.remoting.Channel; +import java.net.InetSocketAddress; +import java.net.URI; import javax.ws.rs.client.Client; import javax.ws.rs.client.WebTarget; -import java.net.InetSocketAddress; -import java.net.URI; + +import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.remoting.Channel; /** * @@ -77,6 +78,11 @@ public WebTarget getWebTarget() { return webTarget; } + @Override + public void close() { + client.close(); + } + /** * Setter method for property webTarget. * diff --git a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyClient.java b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyClient.java index 825a7dd51..c682a8cf0 100644 --- a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyClient.java +++ b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyClient.java @@ -16,28 +16,27 @@ */ package com.alipay.sofa.registry.remoting.jersey; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.UriBuilder; + +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; +import org.glassfish.jersey.jackson.JacksonFeature; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.remoting.CallbackHandler; import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.Client; -import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.client.HttpUrlConnectorProvider; -import org.glassfish.jersey.jackson.JacksonFeature; - -import javax.ws.rs.client.ClientBuilder; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.UriBuilder; -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; /** * @@ -49,7 +48,9 @@ public class JerseyClient implements Client { private static final Logger LOGGER = LoggerFactory .getLogger(JerseyClient.class); private volatile static JerseyClient instance; + private final AtomicReference client = new AtomicReference<>(null); + private Map channels = new HashMap<>(); /** @@ -89,6 +90,22 @@ public Channel connect(URL url) { } } + @Override + public Object sendSync(URL url, Object message, int timeoutMillis) { + return null; + } + + @Override + public Object sendSync(Channel channel, Object message, int timeoutMillis) { + return null; + } + + @Override + public void sendCallback(URL url, Object message, CallbackHandler callbackHandler, + int timeoutMillis) { + + } + private WebTarget getTarget(URL targetUrl) { return getClient().target(getBaseUri(targetUrl)); } @@ -125,24 +142,6 @@ public URI getBaseUri(URL targetUrl) { return uri; } - @Override - public Collection getChannels() { - return null; - } - - @Override - public Channel getChannel(InetSocketAddress remoteAddress) { - Channel c = channels.get(NetUtil.toAddressString(remoteAddress)); - if (c == null) { - return null; - } else { - if (!c.isConnected()) { - connect(new URL(remoteAddress)); - } - } - return c; - } - @Override public Channel getChannel(URL url) { Channel c = channels.get(url.getAddressString()); @@ -156,11 +155,6 @@ public Channel getChannel(URL url) { return c; } - @Override - public List getChannelHandlers() { - return null; - } - @Override public InetSocketAddress getLocalAddress() { return NetUtil.getLocalSocketAddress(); @@ -171,29 +165,9 @@ public void close() { } - @Override - public void close(Channel channel) { - - } - @Override public boolean isClosed() { return false; } - @Override - public void sendOneway(Channel channel, Object message) { - - } - - @Override - public Object sendSync(Channel channel, Object message, int timeoutMillis) { - return null; - } - - @Override - public void sendCallback(Channel channel, Object message, CallbackHandler callbackHandler, - int timeoutMillis) { - - } } \ No newline at end of file diff --git a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyJettyServer.java b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyJettyServer.java index 680384881..0e08653fa 100644 --- a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyJettyServer.java +++ b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/JerseyJettyServer.java @@ -16,14 +16,14 @@ */ package com.alipay.sofa.registry.remoting.jersey; -import com.alipay.sofa.registry.common.model.store.URL; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.remoting.CallbackHandler; -import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.remoting.ChannelHandler; -import com.alipay.sofa.registry.remoting.Server; -import com.alipay.sofa.registry.remoting.jersey.jetty.server.HttpConnectionCustomFactory; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.Collection; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.ws.rs.ProcessingException; + import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.thread.QueuedThreadPool; @@ -35,13 +35,13 @@ import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.server.spi.Container; -import javax.ws.rs.ProcessingException; -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicBoolean; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.CallbackHandler; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.jersey.jetty.server.HttpConnectionCustomFactory; /** * @@ -163,8 +163,8 @@ public Channel getChannel(URL url) { } @Override - public List getChannelHandlers() { - return null; + public void close(Channel channel) { + throw new UnsupportedOperationException("Jersey Server don't support close Channel."); } @Override @@ -186,11 +186,6 @@ public void close() { throw new RuntimeException("Jersey Server has not started!Server Channel has not created!"); } - @Override - public void close(Channel channel) { - throw new UnsupportedOperationException("Jersey Server don't support close Channel."); - } - @Override public boolean isClosed() { if (server != null) { @@ -199,11 +194,6 @@ public boolean isClosed() { return true; } - @Override - public void sendOneway(Channel channel, Object message) { - - } - @Override public Object sendSync(Channel channel, Object message, int timeoutMillis) { return null; @@ -224,4 +214,9 @@ public URI getBaseUri() { return baseUri; } + @Override + public int getChannelCount() { + return 0; + } + } \ No newline at end of file diff --git a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/exchange/JerseyExchange.java b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/exchange/JerseyExchange.java index be001a92b..f73c8b05f 100644 --- a/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/exchange/JerseyExchange.java +++ b/server/remoting/http/src/main/java/com/alipay/sofa/registry/remoting/jersey/exchange/JerseyExchange.java @@ -16,6 +16,13 @@ */ package com.alipay.sofa.registry.remoting.jersey.exchange; +import java.net.URI; +import java.util.concurrent.ConcurrentHashMap; + +import javax.ws.rs.core.UriBuilder; + +import org.glassfish.jersey.server.ResourceConfig; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -24,11 +31,6 @@ import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.remoting.jersey.JerseyClient; import com.alipay.sofa.registry.remoting.jersey.JerseyJettyServer; -import org.glassfish.jersey.server.ResourceConfig; - -import javax.ws.rs.core.UriBuilder; -import java.net.URI; -import java.util.concurrent.ConcurrentHashMap; /** * @@ -51,6 +53,12 @@ public Client connect(String serverType, URL serverUrl, ResourceConfig... channe return jerseyClient; } + @Override + public Client connect(String serverType, int connNum, URL serverUrl, + ResourceConfig... channelHandlers) { + throw new UnsupportedOperationException(); + } + @Override public Server open(URL url, ResourceConfig... resources) { diff --git a/server/remoting/http/src/test/java/com/alipay/sofa/registry/remoting/jersey/JerseyExchangeTest.java b/server/remoting/http/src/test/java/com/alipay/sofa/registry/remoting/jersey/JerseyExchangeTest.java index 22713c6d2..ee005fc98 100644 --- a/server/remoting/http/src/test/java/com/alipay/sofa/registry/remoting/jersey/JerseyExchangeTest.java +++ b/server/remoting/http/src/test/java/com/alipay/sofa/registry/remoting/jersey/JerseyExchangeTest.java @@ -16,19 +16,21 @@ */ package com.alipay.sofa.registry.remoting.jersey; -import com.alipay.sofa.registry.common.model.store.URL; -import com.alipay.sofa.registry.net.NetUtil; -import com.alipay.sofa.registry.remoting.CallbackHandler; -import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.remoting.jersey.exchange.JerseyExchange; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; + +import java.net.InetSocketAddress; +import java.util.concurrent.Executor; + import org.glassfish.jersey.jackson.JacksonFeature; import org.glassfish.jersey.server.ResourceConfig; import org.junit.Assert; import org.junit.Test; -import java.net.InetSocketAddress; - -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.remoting.CallbackHandler; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.jersey.exchange.JerseyExchange; /** * @author xuanbei @@ -51,6 +53,11 @@ public void onCallback(Channel channel, Object message) { @Override public void onException(Channel channel, Throwable exception) { } + + @Override + public Executor getExecutor() { + return null; + } }; JerseyExchange jerseyExchange = new JerseyExchange(); @@ -80,20 +87,11 @@ private void testJerseyJettyServer(URL url, JerseyJettyServer jerseyJettyServer, Assert.assertNull(jerseyJettyServer.getChannels()); Assert.assertNull(jerseyJettyServer.getChannel(new InetSocketAddress(9663))); Assert.assertNull(jerseyJettyServer.getChannel(url)); - Assert.assertNull(jerseyJettyServer.getChannelHandlers()); Assert.assertEquals(new InetSocketAddress(JERSEY_TEST_PORT), jerseyJettyServer.getLocalAddress()); Assert.assertFalse(jerseyJettyServer.isClosed()); - boolean isException = false; - try { - jerseyJettyServer.close(new JerseyChannel()); - } catch (Throwable t) { - isException = true; - } - Assert.assertTrue(isException); jerseyJettyServer.sendCallback(new JerseyChannel(), new Object(), callbackHandler, 1000); - jerseyJettyServer.sendOneway(new JerseyChannel(), new Object()); Assert.assertNull(jerseyJettyServer.sendSync(new JerseyChannel(), new Object(), 1000)); } @@ -101,15 +99,10 @@ private void testJerseyClient(URL url, JerseyClient jerseyClient, CallbackHandler callbackHandler) { Assert.assertEquals(NetUtil.getLocalSocketAddress(), jerseyClient.getLocalAddress()); Assert.assertFalse(jerseyClient.isClosed()); - Assert.assertNull(jerseyClient.getChannels()); - Assert.assertNull(jerseyClient.getChannel(new InetSocketAddress(9663))); Assert.assertNull(jerseyClient.getChannel(url)); - Assert.assertNull(jerseyClient.getChannelHandlers()); - Assert.assertNull(jerseyClient.sendSync(new JerseyChannel(), new Object(), 1000)); + Assert.assertNull(jerseyClient.sendSync(new URL(), new Object(), 1000)); jerseyClient.close(); - jerseyClient.close(new JerseyChannel()); - jerseyClient.sendOneway(new JerseyChannel(), new Object()); - jerseyClient.sendCallback(new JerseyChannel(), new Object(), callbackHandler, 1000); + jerseyClient.sendCallback(new URL(), new Object(), callbackHandler, 1000); } private void testJerseyChannel(JerseyChannel jerseyChannel) { diff --git a/server/remoting/pom.xml b/server/remoting/pom.xml index a42a1f779..d34affeb3 100644 --- a/server/remoting/pom.xml +++ b/server/remoting/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/server/data/pom.xml b/server/server/data/pom.xml index 4e2428f0c..e2f44c074 100644 --- a/server/server/data/pom.xml +++ b/server/server/data/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 @@ -73,6 +73,10 @@ commons-lang commons-lang + + commons-collections + commons-collections + junit junit diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/DataApplication.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/DataApplication.java index 54079bb12..9578b0929 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/DataApplication.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/DataApplication.java @@ -21,6 +21,7 @@ import com.alipay.sofa.registry.server.data.bootstrap.EnableDataServer; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; /** * diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBeanConfiguration.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBeanConfiguration.java index 5087a451b..50284a0e8 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBeanConfiguration.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBeanConfiguration.java @@ -16,24 +16,45 @@ */ package com.alipay.sofa.registry.server.data.bootstrap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.alipay.sofa.registry.server.data.remoting.dataserver.task.LogMetricsTask; +import org.glassfish.jersey.jackson.JacksonFeature; +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + import com.alipay.sofa.registry.remoting.bolt.exchange.BoltExchange; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.remoting.jersey.exchange.JerseyExchange; +import com.alipay.sofa.registry.server.data.cache.CacheDigestTask; import com.alipay.sofa.registry.server.data.cache.DataServerCache; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.cache.LocalDatumStorage; import com.alipay.sofa.registry.server.data.change.DataChangeHandler; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; import com.alipay.sofa.registry.server.data.change.notify.BackUpNotifier; import com.alipay.sofa.registry.server.data.change.notify.IDataChangeNotifier; import com.alipay.sofa.registry.server.data.change.notify.SessionServerNotifier; +import com.alipay.sofa.registry.server.data.change.notify.SnapshotBackUpNotifier; import com.alipay.sofa.registry.server.data.change.notify.TempPublisherNotifier; -import com.alipay.sofa.registry.server.data.correction.LocalDataServerCleanHandler; import com.alipay.sofa.registry.server.data.datasync.AcceptorStore; import com.alipay.sofa.registry.server.data.datasync.SyncDataService; import com.alipay.sofa.registry.server.data.datasync.sync.LocalAcceptorStore; import com.alipay.sofa.registry.server.data.datasync.sync.Scheduler; import com.alipay.sofa.registry.server.data.datasync.sync.StoreServiceFactory; import com.alipay.sofa.registry.server.data.datasync.sync.SyncDataServiceImpl; +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; import com.alipay.sofa.registry.server.data.event.EventCenter; +import com.alipay.sofa.registry.server.data.event.handler.AfterWorkingProcessHandler; import com.alipay.sofa.registry.server.data.event.handler.DataServerChangeEventHandler; import com.alipay.sofa.registry.server.data.event.handler.LocalDataServerChangeEventHandler; import com.alipay.sofa.registry.server.data.event.handler.MetaServerChangeEventHandler; @@ -51,14 +72,18 @@ import com.alipay.sofa.registry.server.data.remoting.dataserver.handler.SyncDataHandler; import com.alipay.sofa.registry.server.data.remoting.dataserver.task.AbstractTask; import com.alipay.sofa.registry.server.data.remoting.dataserver.task.ConnectionRefreshTask; -import com.alipay.sofa.registry.server.data.remoting.dataserver.task.ReNewNodeTask; +import com.alipay.sofa.registry.server.data.remoting.dataserver.task.RenewNodeTask; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.metaserver.DefaultMetaServiceImpl; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import com.alipay.sofa.registry.server.data.remoting.metaserver.MetaServerConnectionFactory; +import com.alipay.sofa.registry.server.data.remoting.metaserver.handler.NotifyProvideDataChangeHandler; import com.alipay.sofa.registry.server.data.remoting.metaserver.handler.ServerChangeHandler; import com.alipay.sofa.registry.server.data.remoting.metaserver.handler.StatusConfirmHandler; +import com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.ProvideDataProcessorManager; +import com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.processor.DatumExpireProvideDataProcessor; import com.alipay.sofa.registry.server.data.remoting.metaserver.task.ConnectionRefreshMetaTask; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.DisconnectEventHandler; @@ -66,25 +91,20 @@ import com.alipay.sofa.registry.server.data.remoting.sessionserver.forward.ForwardServiceImpl; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.ClientOffHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.DataServerConnectionHandler; +import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.DatumSnapshotHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.GetDataHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.GetDataVersionsHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.PublishDataHandler; +import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.RenewDatumHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.SessionServerRegisterHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.handler.UnPublishDataHandler; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; +import com.alipay.sofa.registry.server.data.renew.LocalDataServerCleanHandler; import com.alipay.sofa.registry.server.data.resource.DataDigestResource; import com.alipay.sofa.registry.server.data.resource.HealthResource; +import com.alipay.sofa.registry.server.data.util.DataMetricsThreadPoolExecutor; +import com.alipay.sofa.registry.util.NamedThreadFactory; import com.alipay.sofa.registry.util.PropertySplitter; -import org.glassfish.jersey.jackson.JacksonFeature; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; /** * @@ -111,7 +131,8 @@ public CommonConfig commonConfig() { } @Bean - public DataServerConfig dataServerBootstrapConfig(CommonConfig commonConfig) { + @ConditionalOnMissingBean + public DataServerConfig dataServerConfig(CommonConfig commonConfig) { return new DataServerConfig(commonConfig); } @@ -124,10 +145,39 @@ public DataNodeStatus dataNodeStatus() { public PropertySplitter propertySplitter() { return new PropertySplitter(); } + + } + + @Configuration + public static class DataServerStorageConfiguration { + + @Bean + @ConditionalOnMissingBean + public DatumCache datumCache() { + return new DatumCache(); + } + + @Bean + @ConditionalOnMissingBean + public LocalDatumStorage localDatumStorage() { + return new LocalDatumStorage(); + } + + } + + @Configuration + public static class LogTaskConfigConfiguration { + + @Bean + public CacheDigestTask cacheDigestTask() { + return new CacheDigestTask(); + } + } @Configuration public static class SessionRemotingConfiguration { + @Bean public Exchange jerseyExchange() { return new JerseyExchange(); @@ -174,23 +224,25 @@ public MetaServerConnectionFactory metaServerConnectionFactory() { } @Bean(name = "serverHandlers") - public Collection serverHandlers(DataServerConfig dataServerBootstrapConfig) { + public Collection serverHandlers() { Collection list = new ArrayList<>(); list.add(getDataHandler()); list.add(clientOffHandler()); list.add(getDataVersionsHandler()); - list.add(publishDataProcessor(dataServerBootstrapConfig)); + list.add(publishDataProcessor()); list.add(sessionServerRegisterHandler()); list.add(unPublishDataHandler()); list.add(dataServerConnectionHandler()); + list.add(renewDatumHandler()); + list.add(datumSnapshotHandler()); return list; } @Bean(name = "serverSyncHandlers") - public Collection serverSyncHandlers(DataServerConfig dataServerBootstrapConfig) { + public Collection serverSyncHandlers() { Collection list = new ArrayList<>(); list.add(getDataHandler()); - list.add(publishDataProcessor(dataServerBootstrapConfig)); + list.add(publishDataProcessor()); list.add(unPublishDataHandler()); list.add(notifyFetchDatumHandler()); list.add(notifyOnlineHandler()); @@ -212,6 +264,7 @@ public Collection metaClientHandlers() { Collection list = new ArrayList<>(); list.add(serverChangeHandler()); list.add(statusConfirmHandler()); + list.add(notifyProvideDataChangeHandler()); return list; } @@ -241,8 +294,18 @@ public AbstractServerHandler clientOffHandler() { } @Bean - public AbstractServerHandler publishDataProcessor(DataServerConfig dataServerBootstrapConfig) { - return new PublishDataHandler(dataServerBootstrapConfig); + public AbstractServerHandler datumSnapshotHandler() { + return new DatumSnapshotHandler(); + } + + @Bean + public RenewDatumHandler renewDatumHandler() { + return new RenewDatumHandler(); + } + + @Bean + public AbstractServerHandler publishDataProcessor() { + return new PublishDataHandler(); } @Bean @@ -271,7 +334,8 @@ public AbstractServerHandler syncDataHandler() { } @Bean - public AbstractClientHandler notifyDataSyncHandler() { + @ConditionalOnMissingBean + public NotifyDataSyncHandler notifyDataSyncHandler() { return new NotifyDataSyncHandler(); } @@ -289,6 +353,37 @@ public AbstractClientHandler serverChangeHandler() { public AbstractClientHandler statusConfirmHandler() { return new StatusConfirmHandler(); } + + @Bean + public NotifyProvideDataChangeHandler notifyProvideDataChangeHandler() { + return new NotifyProvideDataChangeHandler(); + } + + @Bean(name = "afterWorkProcessors") + public List afterWorkingProcessors() { + List list = new ArrayList<>(); + list.add(renewDatumHandler()); + list.add(datumLeaseManager()); + list.add(disconnectEventHandler()); + list.add(notifyDataSyncHandler()); + return list; + } + + @Bean + public AfterWorkingProcessHandler afterWorkingProcessHandler() { + return new AfterWorkingProcessHandler(); + } + + @Bean + public DatumLeaseManager datumLeaseManager() { + return new DatumLeaseManager(); + } + + @Bean + public DisconnectEventHandler disconnectEventHandler() { + return new DisconnectEventHandler(); + } + } @Configuration @@ -313,14 +408,21 @@ public BackUpNotifier backUpNotifier() { return new BackUpNotifier(); } + @Bean + public SnapshotBackUpNotifier snapshotBackUpNotifier() { + return new SnapshotBackUpNotifier(); + } + @Bean(name = "dataChangeNotifiers") - public List dataChangeNotifiers(DataServerConfig dataServerBootstrapConfig) { + public List dataChangeNotifiers() { List list = new ArrayList<>(); list.add(sessionServerNotifier()); list.add(tempPublisherNotifier()); list.add(backUpNotifier()); + list.add(snapshotBackUpNotifier()); return list; } + } @Configuration @@ -345,6 +447,7 @@ public Scheduler syncDataScheduler() { public StoreServiceFactory storeServiceFactory() { return new StoreServiceFactory(); } + } @Configuration @@ -380,11 +483,6 @@ public GetSyncDataHandler getSyncDataHandler() { return new GetSyncDataHandler(); } - @Bean - public DisconnectEventHandler disconnectEventHandler() { - return new DisconnectEventHandler(); - } - @Bean public EventCenter eventCenter() { return new EventCenter(); @@ -394,6 +492,7 @@ public EventCenter eventCenter() { public DataChangeEventCenter dataChangeEventCenter() { return new DataChangeEventCenter(); } + } @Configuration @@ -410,8 +509,13 @@ public ConnectionRefreshMetaTask connectionRefreshMetaTask() { } @Bean - public ReNewNodeTask reNewNodeTask() { - return new ReNewNodeTask(); + public RenewNodeTask renewNodeTask() { + return new RenewNodeTask(); + } + + @Bean + public LogMetricsTask logMetricsTask() { + return new LogMetricsTask(); } @Bean(name = "tasks") @@ -419,7 +523,8 @@ public List tasks() { List list = new ArrayList<>(); list.add(connectionRefreshTask()); list.add(connectionRefreshMetaTask()); - list.add(reNewNodeTask()); + list.add(renewNodeTask()); + list.add(logMetricsTask()); return list; } @@ -427,6 +532,7 @@ public List tasks() { public IMetaServerService metaServerService() { return new DefaultMetaServiceImpl(); } + } @Configuration @@ -445,8 +551,72 @@ public HealthResource healthResource() { } @Bean + @ConditionalOnMissingBean public DataDigestResource dataDigestResource() { return new DataDigestResource(); } + + } + + @Configuration + public static class ExecutorConfiguration { + + @Bean(name = "publishProcessorExecutor") + public ThreadPoolExecutor publishProcessorExecutor(DataServerConfig dataServerConfig) { + return new DataMetricsThreadPoolExecutor("PublishProcessorExecutor", + dataServerConfig.getPublishExecutorMinPoolSize(), + dataServerConfig.getPublishExecutorMaxPoolSize(), 300, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(dataServerConfig.getPublishExecutorQueueSize()), + new NamedThreadFactory("DataServer-PublishProcessor-executor", true)); + } + + @Bean(name = "renewDatumProcessorExecutor") + public ThreadPoolExecutor renewDatumProcessorExecutor(DataServerConfig dataServerConfig) { + return new DataMetricsThreadPoolExecutor("RenewDatumProcessorExecutor", + dataServerConfig.getRenewDatumExecutorMinPoolSize(), + dataServerConfig.getRenewDatumExecutorMaxPoolSize(), 300, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(dataServerConfig.getRenewDatumExecutorQueueSize()), + new NamedThreadFactory("DataServer-RenewDatumProcessor-executor", true)); + } + + @Bean(name = "getDataProcessorExecutor") + public ThreadPoolExecutor getDataProcessorExecutor(DataServerConfig dataServerConfig) { + return new DataMetricsThreadPoolExecutor("GetDataProcessorExecutor", + dataServerConfig.getGetDataExecutorMinPoolSize(), + dataServerConfig.getGetDataExecutorMaxPoolSize(), + dataServerConfig.getGetDataExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(dataServerConfig.getGetDataExecutorQueueSize()), + new NamedThreadFactory("DataServer-GetDataProcessor-executor", true)); + } + + @Bean(name = "defaultRequestExecutor") + public ThreadPoolExecutor defaultRequestExecutor(DataServerConfig dataServerConfig) { + return new DataMetricsThreadPoolExecutor("DefaultRequestExecutor", + dataServerConfig.getDefaultRequestExecutorMinSize(), + dataServerConfig.getDefaultRequestExecutorMaxSize(), + dataServerConfig.getDefaultRequestExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(dataServerConfig.getDefaultRequestExecutorQueueSize()), + new NamedThreadFactory("DefaultRequestThread", true) + + ); + } + } + + @Configuration + public static class DataProvideDataConfiguration { + + @Bean + public ProvideDataProcessor provideDataProcessorManager() { + return new ProvideDataProcessorManager(); + } + + @Bean + public ProvideDataProcessor datumExpireProvideDataProcessor(ProvideDataProcessor provideDataProcessorManager) { + ProvideDataProcessor datumExpireProvideDataProcessor = new DatumExpireProvideDataProcessor(); + ((ProvideDataProcessorManager) provideDataProcessorManager) + .addProvideDataProcessor(datumExpireProvideDataProcessor); + return datumExpireProvideDataProcessor; + } + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java index 02bec192f..8cc934baf 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java @@ -16,6 +16,25 @@ */ package com.alipay.sofa.registry.server.data.bootstrap; +import java.lang.annotation.Annotation; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import javax.annotation.Resource; +import javax.ws.rs.Path; +import javax.ws.rs.ext.Provider; + +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -23,25 +42,15 @@ import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.Server; import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.data.cache.CacheDigestTask; import com.alipay.sofa.registry.server.data.datasync.sync.Scheduler; import com.alipay.sofa.registry.server.data.event.EventCenter; import com.alipay.sofa.registry.server.data.event.MetaServerChangeEvent; import com.alipay.sofa.registry.server.data.event.StartTaskEvent; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.ApplicationContext; - -import javax.annotation.Resource; -import javax.ws.rs.Path; -import javax.ws.rs.ext.Provider; -import java.lang.annotation.Annotation; -import java.util.Collection; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; /** * @@ -55,7 +64,7 @@ public class DataServerBootstrap { .getLogger(DataServerBootstrap.class); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private IMetaServerService metaServerService; @@ -78,12 +87,18 @@ public class DataServerBootstrap { @Autowired private EventCenter eventCenter; + @Autowired + private CacheDigestTask cacheDigestTask; + @Resource(name = "serverHandlers") private Collection serverHandlers; @Resource(name = "serverSyncHandlers") private Collection serverSyncHandlers; + @Autowired + private DatumLeaseManager datumLeaseManager; + private Server server; private Server dataSyncServer; @@ -103,7 +118,9 @@ public class DataServerBootstrap { */ public void start() { try { - LOGGER.info("[DataServerBootstrap] begin start server"); + LOGGER.info("begin start server"); + + LOGGER.info("the configuration items are as follows: " + dataServerConfig.toString()); openDataServer(); @@ -113,13 +130,15 @@ public void start() { startRaftClient(); + fetchProviderData(); + startScheduler(); Runtime.getRuntime().addShutdownHook(new Thread(this::doStop)); - LOGGER.info("[DataServerBootstrap] start server success"); + LOGGER.info("start server success"); } catch (Exception e) { - throw new RuntimeException("[DataServerBootstrap] start server error", e); + throw new RuntimeException("start server error", e); } } @@ -127,15 +146,13 @@ private void openDataServer() { try { if (serverForSessionStarted.compareAndSet(false, true)) { server = boltExchange.open(new URL(NetUtil.getLocalAddress().getHostAddress(), - dataServerBootstrapConfig.getPort()), serverHandlers + dataServerConfig.getPort()), serverHandlers .toArray(new ChannelHandler[serverHandlers.size()])); - LOGGER.info("Data server for session started! port:{}", - dataServerBootstrapConfig.getPort()); + LOGGER.info("Data server for session started! port:{}", dataServerConfig.getPort()); } } catch (Exception e) { serverForSessionStarted.set(false); - LOGGER - .error("Data server start error! port:{}", dataServerBootstrapConfig.getPort(), e); + LOGGER.error("Data server start error! port:{}", dataServerConfig.getPort(), e); throw new RuntimeException("Data server start error!", e); } } @@ -144,15 +161,15 @@ private void openDataSyncServer() { try { if (serverForDataSyncStarted.compareAndSet(false, true)) { dataSyncServer = boltExchange.open(new URL(NetUtil.getLocalAddress() - .getHostAddress(), dataServerBootstrapConfig.getSyncDataPort()), - serverSyncHandlers.toArray(new ChannelHandler[serverSyncHandlers.size()])); + .getHostAddress(), dataServerConfig.getSyncDataPort()), serverSyncHandlers + .toArray(new ChannelHandler[serverSyncHandlers.size()])); LOGGER.info("Data server for sync started! port:{}", - dataServerBootstrapConfig.getSyncDataPort()); + dataServerConfig.getSyncDataPort()); } } catch (Exception e) { serverForDataSyncStarted.set(false); LOGGER.error("Data sync server start error! port:{}", - dataServerBootstrapConfig.getSyncDataPort(), e); + dataServerConfig.getSyncDataPort(), e); throw new RuntimeException("Data sync server start error!", e); } } @@ -162,15 +179,15 @@ private void openHttpServer() { if (httpServerStarted.compareAndSet(false, true)) { bindResourceConfig(); httpServer = jerseyExchange.open( - new URL(NetUtil.getLocalAddress().getHostAddress(), dataServerBootstrapConfig + new URL(NetUtil.getLocalAddress().getHostAddress(), dataServerConfig .getHttpServerPort()), new ResourceConfig[] { jerseyResourceConfig }); LOGGER.info("Open http server port {} success!", - dataServerBootstrapConfig.getHttpServerPort()); + dataServerConfig.getHttpServerPort()); } } catch (Exception e) { httpServerStarted.set(false); - LOGGER.error("Open http server port {} error!", - dataServerBootstrapConfig.getHttpServerPort(), e); + LOGGER + .error("Open http server port {} error!", dataServerConfig.getHttpServerPort(), e); throw new RuntimeException("Open http server error!", e); } } @@ -178,15 +195,35 @@ private void openHttpServer() { private void startRaftClient() { metaServerService.startRaftClient(); eventCenter.post(new MetaServerChangeEvent(metaServerService.getMetaServerMap())); - LOGGER.info("[DataServerBootstrap] raft client started!Leader is {}", - metaServerService.getLeader()); + LOGGER.info("raft client started!Leader is {}", metaServerService.getLeader()); + } + + private void fetchProviderData() { + ProvideData provideData = metaServerService + .fetchData(ValueConstants.ENABLE_DATA_DATUM_EXPIRE); + if (provideData == null || provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER + .info("Fetch enableDataDatumExpire but no data existed, current config not change!"); + return; + } + boolean enableDataDatumExpire = Boolean.parseBoolean((String) provideData.getProvideData() + .getObject()); + LOGGER.info("Fetch enableDataDatumExpire {} success!", enableDataDatumExpire); + datumLeaseManager.setRenewEnable(enableDataDatumExpire); } private void startScheduler() { try { if (schedulerStarted.compareAndSet(false, true)) { syncDataScheduler.startScheduler(); - eventCenter.post(StartTaskEvent.getInstance()); + // start all startTask except correction task + eventCenter.post(new StartTaskEvent( + Arrays.stream(StartTaskTypeEnum.values()).filter(type -> type != StartTaskTypeEnum.RENEW) + .collect(Collectors.toSet()))); + + //start dump log + cacheDigestTask.start(); } } catch (Exception e) { schedulerStarted.set(false); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java index 4d801b711..66dadaed3 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java @@ -16,15 +16,18 @@ */ package com.alipay.sofa.registry.server.data.bootstrap; -import com.alipay.sofa.registry.net.NetUtil; -import org.springframework.boot.context.properties.ConfigurationProperties; - import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import com.alipay.sofa.registry.net.NetUtil; + /** * * @@ -34,10 +37,11 @@ @ConfigurationProperties(prefix = DataServerConfig.PRE_FIX) public class DataServerConfig { - public static final String PRE_FIX = "data.server"; + public static final String PRE_FIX = "data.server"; - public static final String IP = NetUtil.getLocalAddress() - .getHostAddress(); + public static final String IP = NetUtil + .getLocalAddress() + .getHostAddress(); private int port; @@ -55,45 +59,80 @@ public class DataServerConfig { private int clientOffDelayMs; + private int notifyTempDataIntervalMs; + private int rpcTimeout; private CommonConfig commonConfig; - private Set metaIps = null; + private Set metaIps = null; + + private int storeNodes = 3; + + private int numberOfReplicas = 1000; + + private long localDataServerCleanDelay = 1000 * 60 * 30; + + private int getDataExecutorMinPoolSize = 80; + + private int getDataExecutorMaxPoolSize = 400; + + private int getDataExecutorQueueSize = 10000; + + private long getDataExecutorKeepAliveTime = 60; + + private int notifyDataSyncExecutorMinPoolSize = 80; + + private int notifyDataSyncExecutorMaxPoolSize = 400; + + private int notifyDataSyncExecutorQueueSize = 700; + + private long notifyDataSyncExecutorKeepAliveTime = 60; + + private long notifySessionRetryFirstDelay = 3000; + + private long notifySessionRetryIncrementDelay = 3000; + + private int notifySessionRetryTimes = 5; + + private int publishExecutorMinPoolSize = 200; - private int storeNodes = 3; + private int publishExecutorMaxPoolSize = 400; - private int numberOfReplicas = 1000; + private int publishExecutorQueueSize = 10000; - private long localDataServerCleanDelay = 1000 * 60 * 30; + private int renewDatumExecutorMinPoolSize = 100; - private int getDataExecutorMinPoolSize = 80; + private int renewDatumExecutorMaxPoolSize = 400; - private int getDataExecutorMaxPoolSize = 400; + private int renewDatumExecutorQueueSize = 100000; - private int getDataExecutorQueueSize = 10000; + private int datumTimeToLiveSec = 900; - private long getDataExecutorKeepAliveTime = 60; + private int datumLeaseManagerExecutorThreadSize = 1; - private int notifyDataSyncExecutorMinPoolSize = 80; + private int datumLeaseManagerExecutorQueueSize = 1000000; - private int notifyDataSyncExecutorMaxPoolSize = 400; + private int sessionServerNotifierRetryExecutorThreadSize = 10; - private int notifyDataSyncExecutorQueueSize = 700; + private int sessionServerNotifierRetryExecutorQueueSize = 10000; - private long notifyDataSyncExecutorKeepAliveTime = 60; + private int defaultRequestExecutorMinSize = 20; - private long notifySessionRetryFirstDelay = 1000; + private int defaultRequestExecutorMaxSize = 400; - private long notifySessionRetryIncrementDelay = 1000; + private int defaultRequestExecutorQueueSize = 600; + private long defaultRequestExecutorKeepAliveTime = 60; - private int notifySessionRetryTimes = 10; + private int logMetricsFixedDelay = 30; - private int publishExecutorMinPoolSize = 80; + private int renewEnableDelaySec = 30; - private int publishExecutorMaxPoolSize = 400; + private int dataSyncDelayTimeout = 1000; - private int publishExecutorQueueSize = 10000; + private int dataSyncNotifyRetry = 3; + + private int sessionDisconnectDelayMs = 30000; /** * constructor @@ -103,10 +142,86 @@ public DataServerConfig(CommonConfig commonConfig) { this.commonConfig = commonConfig; } + public boolean isLocalDataCenter(String dataCenter) { + return commonConfig.getLocalDataCenter().equals(dataCenter); + } + + /** + * Getter method for property renewEnableDelaySec. + * + * @return property value of renewEnableDelaySec + */ + public int getRenewEnableDelaySec() { + return renewEnableDelaySec; + } + + /** + * Setter method for property renewEnableDelaySec . + * + * @param renewEnableDelaySec value to be assigned to property renewEnableDelaySec + */ + public void setRenewEnableDelaySec(int renewEnableDelaySec) { + this.renewEnableDelaySec = renewEnableDelaySec; + } + public String getLocalDataCenter() { return commonConfig.getLocalDataCenter(); } + /** + * Getter method for property renewDatumExecutorMinPoolSize. + * + * @return property value of renewDatumExecutorMinPoolSize + */ + public int getRenewDatumExecutorMinPoolSize() { + return renewDatumExecutorMinPoolSize; + } + + /** + * Setter method for property renewDatumExecutorMinPoolSize . + * + * @param renewDatumExecutorMinPoolSize value to be assigned to property renewDatumExecutorMinPoolSize + */ + public void setRenewDatumExecutorMinPoolSize(int renewDatumExecutorMinPoolSize) { + this.renewDatumExecutorMinPoolSize = renewDatumExecutorMinPoolSize; + } + + /** + * Getter method for property renewDatumExecutorMaxPoolSize. + * + * @return property value of renewDatumExecutorMaxPoolSize + */ + public int getRenewDatumExecutorMaxPoolSize() { + return renewDatumExecutorMaxPoolSize; + } + + /** + * Setter method for property renewDatumExecutorMaxPoolSize . + * + * @param renewDatumExecutorMaxPoolSize value to be assigned to property renewDatumExecutorMaxPoolSize + */ + public void setRenewDatumExecutorMaxPoolSize(int renewDatumExecutorMaxPoolSize) { + this.renewDatumExecutorMaxPoolSize = renewDatumExecutorMaxPoolSize; + } + + /** + * Getter method for property renewDatumExecutorQueueSize. + * + * @return property value of renewDatumExecutorQueueSize + */ + public int getRenewDatumExecutorQueueSize() { + return renewDatumExecutorQueueSize; + } + + /** + * Setter method for property renewDatumExecutorQueueSize . + * + * @param renewDatumExecutorQueueSize value to be assigned to property renewDatumExecutorQueueSize + */ + public void setRenewDatumExecutorQueueSize(int renewDatumExecutorQueueSize) { + this.renewDatumExecutorQueueSize = renewDatumExecutorQueueSize; + } + /** * Getter method for property port. * @@ -233,6 +348,24 @@ public void setNotifyIntervalMs(int notifyIntervalMs) { this.notifyIntervalMs = notifyIntervalMs; } + /** + * Getter method for property notifyTempDataIntervalMs. + * + * @return property value of notifyTempDataIntervalMs + */ + public int getNotifyTempDataIntervalMs() { + return notifyTempDataIntervalMs; + } + + /** + * Setter method for property notifyTempDataIntervalMs. + * + * @param notifyTempDataIntervalMs value to be assigned to property notifyTempDataIntervalMs + */ + public void setNotifyTempDataIntervalMs(int notifyTempDataIntervalMs) { + this.notifyTempDataIntervalMs = notifyTempDataIntervalMs; + } + /** * Getter method for property rpcTimeout. * @@ -574,7 +707,8 @@ public Set getMetaServerIpAddresses() { if (localDataCenter != null && !localDataCenter.isEmpty()) { Collection metas = metaMap.get(localDataCenter); if (metas != null && !metas.isEmpty()) { - metaIps = metas.stream().map(NetUtil::getIPAddressFromDomain).collect(Collectors.toSet()); + metaIps = metas.stream().map(NetUtil::getIPAddressFromDomain) + .collect(Collectors.toSet()); } } } @@ -599,4 +733,194 @@ public int getNotifySessionRetryTimes() { public void setNotifySessionRetryTimes(int notifySessionRetryTimes) { this.notifySessionRetryTimes = notifySessionRetryTimes; } + + /** + * Getter method for property datumTimeToLiveSec. + * + * @return property value of datumTimeToLiveSec + */ + public int getDatumTimeToLiveSec() { + return datumTimeToLiveSec; + } + + /** + * Setter method for property datumTimeToLiveSec . + * + * @param datumTimeToLiveSec value to be assigned to property datumTimeToLiveSec + */ + public void setDatumTimeToLiveSec(int datumTimeToLiveSec) { + this.datumTimeToLiveSec = datumTimeToLiveSec; + } + + /** + * Getter method for property datumLeaseManagerExecutorQueueSize. + * + * @return property value of datumLeaseManagerExecutorQueueSize + */ + public int getDatumLeaseManagerExecutorQueueSize() { + return datumLeaseManagerExecutorQueueSize; + } + + /** + * Setter method for property datumLeaseManagerExecutorQueueSize . + * + * @param datumLeaseManagerExecutorQueueSize value to be assigned to property datumLeaseManagerExecutorQueueSize + */ + public void setDatumLeaseManagerExecutorQueueSize(int datumLeaseManagerExecutorQueueSize) { + this.datumLeaseManagerExecutorQueueSize = datumLeaseManagerExecutorQueueSize; + } + + /** + * Getter method for property datumLeaseManagerExecutorThreadSize. + * + * @return property value of datumLeaseManagerExecutorThreadSize + */ + public int getDatumLeaseManagerExecutorThreadSize() { + return datumLeaseManagerExecutorThreadSize; + } + + /** + * Setter method for property datumLeaseManagerExecutorThreadSize . + * + * @param datumLeaseManagerExecutorThreadSize value to be assigned to property datumLeaseManagerExecutorThreadSize + */ + public void setDatumLeaseManagerExecutorThreadSize(int datumLeaseManagerExecutorThreadSize) { + this.datumLeaseManagerExecutorThreadSize = datumLeaseManagerExecutorThreadSize; + } + + /** + * Getter method for property sessionServerNotifierRetryExecutorThreadSize. + * + * @return property value of sessionServerNotifierRetryExecutorThreadSize + */ + public int getSessionServerNotifierRetryExecutorThreadSize() { + return sessionServerNotifierRetryExecutorThreadSize; + } + + /** + * Setter method for property sessionServerNotifierRetryExecutorThreadSize . + * + * @param sessionServerNotifierRetryExecutorThreadSize value to be assigned to property sessionServerNotifierRetryExecutorThreadSize + */ + public void setSessionServerNotifierRetryExecutorThreadSize(int sessionServerNotifierRetryExecutorThreadSize) { + this.sessionServerNotifierRetryExecutorThreadSize = sessionServerNotifierRetryExecutorThreadSize; + } + + /** + * Getter method for property sessionServerNotifierRetryExecutorQueueSize. + * + * @return property value of sessionServerNotifierRetryExecutorQueueSize + */ + public int getSessionServerNotifierRetryExecutorQueueSize() { + return sessionServerNotifierRetryExecutorQueueSize; + } + + /** + * Setter method for property sessionServerNotifierRetryExecutorQueueSize . + * + * @param sessionServerNotifierRetryExecutorQueueSize value to be assigned to property sessionServerNotifierRetryExecutorQueueSize + */ + public void setSessionServerNotifierRetryExecutorQueueSize(int sessionServerNotifierRetryExecutorQueueSize) { + this.sessionServerNotifierRetryExecutorQueueSize = sessionServerNotifierRetryExecutorQueueSize; + } + + /** + * Getter method for property dataSyncDelayTimeout. + * + * @return property value of dataSyncDelayTimeout + */ + public int getDataSyncDelayTimeout() { + return dataSyncDelayTimeout; + } + + /** + * Setter method for property dataSyncDelayTimeout . + * + * @param dataSyncDelayTimeout value to be assigned to property dataSyncDelayTimeout + */ + public void setDataSyncDelayTimeout(int dataSyncDelayTimeout) { + this.dataSyncDelayTimeout = dataSyncDelayTimeout; + } + + /** + * Getter method for property dataSyncNotifyRetry. + * + * @return property value of dataSyncNotifyRetry + */ + public int getDataSyncNotifyRetry() { + return dataSyncNotifyRetry; + } + + /** + * Setter method for property dataSyncNotifyRetry . + * + * @param dataSyncNotifyRetry value to be assigned to property dataSyncNotifyRetry + */ + public void setDataSyncNotifyRetry(int dataSyncNotifyRetry) { + this.dataSyncNotifyRetry = dataSyncNotifyRetry; + } + + /** + * Getter method for property sessionDisconnectDelayMs. + * + * @return property value of sessionDisconnectDelayMs + */ + public int getSessionDisconnectDelayMs() { + return sessionDisconnectDelayMs; + } + + /** + * Setter method for property sessionDisconnectDelayMs. + * + * @param sessionDisconnectDelayMs value to be assigned to property sessionDisconnectDelayMs + */ + public void setSessionDisconnectDelayMs(int sessionDisconnectDelayMs) { + this.sessionDisconnectDelayMs = sessionDisconnectDelayMs; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } + + public int getDefaultRequestExecutorMinSize() { + return defaultRequestExecutorMinSize; + } + + public void setDefaultRequestExecutorMinSize(int defaultRequestExecutorMinSize) { + this.defaultRequestExecutorMinSize = defaultRequestExecutorMinSize; + } + + public int getDefaultRequestExecutorMaxSize() { + return defaultRequestExecutorMaxSize; + } + + public void setDefaultRequestExecutorMaxSize(int defaultRequestExecutorMaxSize) { + this.defaultRequestExecutorMaxSize = defaultRequestExecutorMaxSize; + } + + public int getDefaultRequestExecutorQueueSize() { + return defaultRequestExecutorQueueSize; + } + + public void setDefaultRequestExecutorQueueSize(int defaultRequestExecutorQueueSize) { + this.defaultRequestExecutorQueueSize = defaultRequestExecutorQueueSize; + } + + public long getDefaultRequestExecutorKeepAliveTime() { + return defaultRequestExecutorKeepAliveTime; + } + + public void setDefaultRequestExecutorKeepAliveTime(long defaultRequestExecutorKeepAliveTime) { + this.defaultRequestExecutorKeepAliveTime = defaultRequestExecutorKeepAliveTime; + } + + public int getLogMetricsFixedDelay() { + return logMetricsFixedDelay; + } + + public void setLogMetricsFixedDelay(int logMetricsFixedDelay) { + this.logMetricsFixedDelay = logMetricsFixedDelay; + } + } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/CacheDigestTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/CacheDigestTask.java index 5b09ddb31..77b7328b1 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/CacheDigestTask.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/CacheDigestTask.java @@ -16,19 +16,21 @@ */ package com.alipay.sofa.registry.server.data.cache; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; + import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.util.NamedThreadFactory; -import org.springframework.util.CollectionUtils; - -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * @@ -39,20 +41,23 @@ public class CacheDigestTask { private static final Logger LOGGER = LoggerFactory.getLogger(CacheDigestTask.class); + @Autowired + private DatumCache datumCache; + /** * */ public void start() { - ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("CacheDigestTask")); + ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1, + new NamedThreadFactory("CacheDigestTask")); executor.scheduleAtFixedRate(() -> { try { - Map> allMap = DatumCache.getAll(); + Map> allMap = datumCache.getAll(); if (!allMap.isEmpty()) { for (Entry> dataCenterEntry : allMap.entrySet()) { String dataCenter = dataCenterEntry.getKey(); Map datumMap = dataCenterEntry.getValue(); - LOGGER.info("[CacheDigestTask] size of datum in {} is {}", - dataCenter, datumMap.size()); + LOGGER.info("[CacheDigestTask] size of datum in {} is {}", dataCenter, datumMap.size()); for (Entry dataInfoEntry : datumMap.entrySet()) { String dataInfoId = dataInfoEntry.getKey(); Datum data = dataInfoEntry.getValue(); @@ -63,14 +68,12 @@ public void start() { pubStr.append(logPublisher(publisher)).append(";"); } } - LOGGER.info( - "[Datum] dataInfoId={}, version={}, dataCenter={}, publishers=[{}]", - dataInfoId, data.getVersion(), dataCenter, pubStr.toString()); + LOGGER.info("[Datum]{},{},{},[{}]", dataInfoId, + data.getVersion(), dataCenter, pubStr.toString()); } int pubCount = datumMap.values().stream().map(Datum::getPubMap) .filter(map -> map != null && !map.isEmpty()).mapToInt(Map::size).sum(); - LOGGER.info("[CacheDigestTask] size of publisher in {} is {}", - dataCenter, pubCount); + LOGGER.info("[CacheDigestTask] size of publisher in {} is {}", dataCenter, pubCount); } } else { LOGGER.info("[CacheDigestTask] datum cache is empty"); @@ -79,16 +82,15 @@ public void start() { } catch (Throwable t) { LOGGER.error("[CacheDigestTask] cache digest error", t); } - }, 30, 300, TimeUnit.SECONDS); + }, 30, 600, TimeUnit.SECONDS); } private String logPublisher(Publisher publisher) { if (publisher != null) { URL url = publisher.getSourceAddress(); String urlStr = url != null ? url.getAddressString() : "null"; - return String.format("dataInfoId=%s, version=%s, host=%s, registerId=%s", - publisher.getDataInfoId(), publisher.getVersion(), urlStr, - publisher.getRegisterId()); + return String.format("%s,%s,%s,%s", publisher.getRegisterId(), + publisher.getRegisterTimestamp(), urlStr, publisher.getVersion()); } return ""; } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DataServerCache.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DataServerCache.java index 163d633ff..64d3d9ee0 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DataServerCache.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DataServerCache.java @@ -16,19 +16,9 @@ */ package com.alipay.sofa.registry.server.data.cache; -import com.alipay.sofa.registry.common.model.metaserver.DataNode; -import com.alipay.sofa.registry.consistency.hash.ConsistentHash; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; -import com.alipay.sofa.registry.server.data.node.DataNodeStatus; -import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; -import org.springframework.beans.factory.annotation.Autowired; - import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -36,6 +26,21 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.metaserver.DataNode; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.consistency.hash.ConsistentHash; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent.FromType; +import com.alipay.sofa.registry.server.data.event.handler.AfterWorkingProcessHandler; +import com.alipay.sofa.registry.server.data.node.DataNodeStatus; +import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + /** * cache of dataservers * @@ -53,8 +58,13 @@ public class DataServerCache { @Autowired private DataServerConfig dataServerConfig; + @Autowired + private AfterWorkingProcessHandler afterWorkingProcessHandler; + + /** current dataServer list and version */ private volatile DataServerChangeItem dataServerChangeItem = new DataServerChangeItem(); + /** new input dataServer list and version */ private volatile DataServerChangeItem newDataServerChangeItem = new DataServerChangeItem(); private final AtomicBoolean HAS_NOTIFY_ALL = new AtomicBoolean( @@ -72,7 +82,7 @@ public class DataServerCache { * @param newItem * @return changedMap(datacenter, serverIp) */ - public Map> compareAndSet(DataServerChangeItem newItem) { + public Map> compareAndSet(DataServerChangeItem newItem, FromType fromType) { synchronized (DataServerCache.class) { // versionMap: datacenter -> version Map newVersionMap = newItem.getVersionMap(); @@ -103,8 +113,8 @@ public Map> compareAndSet(DataServerChangeItem newItem) { if (isTheSame.get()) { LOGGER .info( - "current process map has a same version as change map,this change will be ignored!process version={},get version={}", - currentNewVersionMap, newVersionMap); + "current process map has a same version as change map,this change will be ignored!process version={},get version={},from={}", + currentNewVersionMap, newVersionMap, fromType); return new HashMap<>(); } } @@ -132,7 +142,8 @@ public Map> compareAndSet(DataServerChangeItem newItem) { init(newVersionMap.get(dataServerConfig.getLocalDataCenter())); } if (!changedMap.isEmpty()) { - LOGGER.info("old server map = {}", dataServerChangeItem.getServerMap()); + LOGGER.info("old server map = {},from={}", dataServerChangeItem.getServerMap(), + fromType); LOGGER.info("new server map = {}", newServerMap); LOGGER.info("new server version map = {}", newVersionMap); LOGGER.info("status map = {}", nodeStatusMap); @@ -242,13 +253,15 @@ private void updateDataServerStatus() { Map map = nodeStatusMap.get(curVersion.get()); if (map != null) { Set ips = map.keySet(); - if (!ips.containsAll(newDataServerChangeItem.getServerMap() - .get(dataServerConfig.getLocalDataCenter()).keySet())) { - LOGGER.info( - "nodeStatusMap not contains all push list,nodeStatusMap {} push {}", - nodeStatusMap, - newDataServerChangeItem.getServerMap() - .get(dataServerConfig.getLocalDataCenter()).keySet()); + Set itemIps = newDataServerChangeItem.getServerMap() + .get(dataServerConfig.getLocalDataCenter()).keySet(); + if (!ips.containsAll(itemIps)) { + + LOGGER + .info( + "nodeStatusMap not contains all push list,nodeStatusMap {},push {},diff1{},diff2{}", + nodeStatusMap, itemIps, Sets.difference(ips, itemIps), + Sets.difference(itemIps, ips)); return; } } else { @@ -263,8 +276,16 @@ private void updateDataServerStatus() { dataNodeStatus.setStatus(LocalServerStatusEnum.WORKING); + //after working update current dataCenter list to old DataServerChangeItem + updateItem( + newDataServerChangeItem.getServerMap().get(dataServerConfig.getLocalDataCenter()), + newVersion, dataServerConfig.getLocalDataCenter()); + //after working status,must clean this map,because calculate backupTriad need add not working node,see LocalDataServerChangeEventHandler getToBeSyncMap resetStatusMapToWorking(); + + //after working process + afterWorkingProcessHandler.afterWorkingProcess(); } } @@ -351,25 +372,37 @@ public Long getDataCenterNewVersion(String dataCenter) { } } - public BackupTriad calculateOldBackupTriad(String dataInfoId, String dataCenter, - DataServerConfig dataServerBootstrapConfig) { + /** + * calculate ConsistentHash base current data server list + * do not return null, otherwise will lead to unexpected consequences + * + * 20200211 update: bugfix: empty dataServerList cause NPE because calculateOldConsistentHash return null + */ + public ConsistentHash calculateOldConsistentHash(String dataCenter) { Map> dataServerMap = dataServerChangeItem.getServerMap(); Map dataNodeMap = dataServerMap.get(dataCenter); + Collection dataServerNodes; if (dataNodeMap != null && !dataNodeMap.isEmpty()) { - - Collection dataServerNodes = dataNodeMap.values(); - - ConsistentHash consistentHash = new ConsistentHash<>( - dataServerBootstrapConfig.getNumberOfReplicas(), dataServerNodes); - - List list = consistentHash.getNUniqueNodesFor(dataInfoId, - dataServerBootstrapConfig.getStoreNodes()); - - return new BackupTriad(dataInfoId, list); + dataServerNodes = dataNodeMap.values(); } else { - LOGGER.warn("Calculate Old BackupTriad,old dataServer list is empty!"); - return null; + dataServerNodes = Lists.newArrayList(new DataNode(new URL(DataServerConfig.IP), + dataCenter)); + LOGGER + .error("[calculateOldConsistentHash] Old dataServer list is empty, add on the local IP"); } + ConsistentHash consistentHash = new ConsistentHash<>( + dataServerConfig.getNumberOfReplicas(), dataServerNodes); + + return consistentHash; + } + + /** + * get all datacenters + * + * @return + */ + public Set getAllDataCenters() { + return newDataServerChangeItem.getVersionMap().keySet(); } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumCache.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumCache.java index 6b17f7702..53b577ada 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumCache.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumCache.java @@ -16,41 +16,28 @@ */ package com.alipay.sofa.registry.server.data.cache; -import com.alipay.sofa.registry.common.model.dataserver.Datum; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; -import org.springframework.util.StringUtils; - import java.util.HashMap; -import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; /** * cache of datum, providing query function to the upper module * + * @author kezhu.wukz * @author qian.lqlq * @version $Id: DatumCache.java, v 0.1 2017-12-06 20:50 qian.lqlq Exp $ */ public class DatumCache { - public static final long ERROR_DATUM_VERSION = -2L; - - /** - * row: dataCenter - * column: dataInfoId - * value: datum - */ - private static final Map> DATUM_MAP = new ConcurrentHashMap<>(); - - /** - * row: ip:port - * column: registerId - * value: publisher - */ - private static final Map> CLIENT_PUB_MAP = new ConcurrentHashMap<>(); + @Autowired + private DatumStorage localDatumStorage; /** * get datum by specific dataCenter and dataInfoId @@ -59,29 +46,38 @@ public class DatumCache { * @param dataInfoId * @return */ - public static Datum get(String dataCenter, String dataInfoId) { - if (DATUM_MAP.containsKey(dataCenter)) { - Map map = DATUM_MAP.get(dataCenter); - if (map.containsKey(dataInfoId)) { - return map.get(dataInfoId); - } - } - return null; + public Datum get(String dataCenter, String dataInfoId) { + return localDatumStorage.get(dataCenter, dataInfoId); } /** - * get datum of all datercenters by dataInfoId + * get datum of all data centers by dataInfoId * * @param dataInfoId * @return */ - public static Map get(String dataInfoId) { + public Map get(String dataInfoId) { Map datumMap = new HashMap<>(); - DATUM_MAP.forEach((dataCenter, datums) -> { - if (datums.containsKey(dataInfoId)) { - datumMap.put(dataCenter, datums.get(dataInfoId)); - } - }); + + //local + Map localDataCenterToMap = localDatumStorage.get(dataInfoId); + datumMap.putAll(localDataCenterToMap); + + return datumMap; + } + + /** + * get datum of all data centers by dataInfoId + * + * @param dataInfoId + * @return + */ + public Map getVersions(String dataInfoId) { + Map datumMap = new HashMap<>(); + + //local + Map localVersions = localDatumStorage.getVersions(dataInfoId); + datumMap.putAll(localVersions); return datumMap; } @@ -93,12 +89,12 @@ public static Map get(String dataInfoId) { * @param dataInfoId * @return */ - public static Map getDatumGroupByDataCenter(String dataCenter, String dataInfoId) { + public Map getDatumGroupByDataCenter(String dataCenter, String dataInfoId) { Map map = new HashMap<>(); if (StringUtils.isEmpty(dataCenter)) { - map = DatumCache.get(dataInfoId); + map = this.get(dataInfoId); } else { - Datum datum = DatumCache.get(dataCenter, dataInfoId); + Datum datum = this.get(dataCenter, dataInfoId); if (datum != null) { map.put(dataCenter, datum); } @@ -111,18 +107,25 @@ public static Map getDatumGroupByDataCenter(String dataCenter, St * * @return */ - public static Map> getAll() { - return DATUM_MAP; + public Map> getAll() { + return localDatumStorage.getAll(); } /** * * - * @param host + * @param connectId * @return */ - public static Map getByHost(String host) { - return CLIENT_PUB_MAP.getOrDefault(host, null); + public Map getByConnectId(String connectId) { + return localDatumStorage.getByConnectId(connectId); + } + + /** + * get own publishers by connectId + */ + public Map getOwnByConnectId(String connectId) { + return localDatumStorage.getOwnByConnectId(connectId); } /** @@ -132,56 +135,8 @@ public static Map getByHost(String host) { * @param datum * @return the last version before datum changed, if datum is not exist, return null */ - public static MergeResult putDatum(DataChangeTypeEnum changeType, Datum datum) { - MergeResult mergeResult; - String dataCenter = datum.getDataCenter(); - String dataInfoId = datum.getDataInfoId(); - Map map = DATUM_MAP.get(dataCenter); - if (map == null) { - map = new ConcurrentHashMap<>(); - Map ret = DATUM_MAP.putIfAbsent(dataCenter, map); - if (ret != null) { - map = ret; - } - } - - //first put UnPublisher datum(dataId group instanceId is null),can not add to cache - if (datum.getDataId() == null && map.get(dataInfoId) == null) { - mergeResult = new MergeResult(ERROR_DATUM_VERSION, false); - return mergeResult; - } - - Datum ret = map.putIfAbsent(dataInfoId, datum); - if (ret == null) { - Set> entries = datum.getPubMap().entrySet(); - Iterator> iterator = entries.iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - Publisher publisher = entry.getValue(); - if (!(publisher instanceof UnPublisher)) { - String registerId = publisher.getRegisterId(); - Map clientRegisterMap = new ConcurrentHashMap<>(); - clientRegisterMap.put(registerId, publisher); - Map retMap = CLIENT_PUB_MAP.putIfAbsent(publisher - .getSourceAddress().getAddressString(), clientRegisterMap); - if (retMap != null) { - retMap.putAll(clientRegisterMap); - } - } else { - //first put to cache,UnPublisher data must remove,not so got error pub data exist - iterator.remove(); - } - } - mergeResult = new MergeResult(null, true); - } else { - if (changeType == DataChangeTypeEnum.MERGE) { - mergeResult = mergeDatum(datum); - } else { - Long lastVersion = coverDatum(datum); - mergeResult = new MergeResult(lastVersion, true); - } - } - return mergeResult; + public MergeResult putDatum(DataChangeTypeEnum changeType, Datum datum) { + return localDatumStorage.putDatum(changeType, datum); } /** @@ -190,120 +145,25 @@ public static MergeResult putDatum(DataChangeTypeEnum changeType, Datum datum) { * @param dataInfoId * @return */ - public static boolean cleanDatum(String dataCenter, String dataInfoId) { - - Map datumMap = DATUM_MAP.get(dataCenter); - if (datumMap != null) { - Datum cacheDatum = datumMap.remove(dataInfoId); - if (cacheDatum != null) { - Map cachePubMap = cacheDatum.getPubMap(); - - for (Entry cachePubEntry : cachePubMap.entrySet()) { - String registerId = cachePubEntry.getKey(); - Publisher cachePub = cachePubEntry.getValue(); - //remove from cache - if (cachePub != null) { - cachePubMap.remove(registerId); - CLIENT_PUB_MAP.get(cachePub.getSourceAddress().getAddressString()).remove( - registerId); - } - } - return true; - } - } - return false; + public boolean cleanDatum(String dataCenter, String dataInfoId) { + return localDatumStorage.cleanDatum(dataCenter, dataInfoId); } /** - * merge datum in cache - * - * @param datum - * @return + * cover datum by snapshot */ - private static MergeResult mergeDatum(Datum datum) { - boolean isChanged = false; - Datum cacheDatum = DATUM_MAP.get(datum.getDataCenter()).get(datum.getDataInfoId()); - Map cachePubMap = cacheDatum.getPubMap(); - Map pubMap = datum.getPubMap(); - for (Entry pubEntry : pubMap.entrySet()) { - String registerId = pubEntry.getKey(); - Publisher pub = pubEntry.getValue(); - Publisher cachePub = cachePubMap.get(registerId); - if (pub instanceof UnPublisher) { - //remove from cache - if (cachePub != null - && pub.getRegisterTimestamp() > cachePub.getRegisterTimestamp()) { - cachePubMap.remove(registerId); - CLIENT_PUB_MAP.get(cachePub.getSourceAddress().getAddressString()).remove( - registerId); - isChanged = true; - } - } else { - String pubAddr = pub.getSourceAddress().getAddressString(); - long version = pub.getVersion(); - long cacheVersion = cachePub == null ? 0L : cachePub.getVersion(); - String cachePubAddr = cachePub == null ? "" : cachePub.getSourceAddress() - .getAddressString(); - if (cacheVersion <= version) { - cachePubMap.put(registerId, pub); - if (cacheVersion < version || !pubAddr.equals(cachePubAddr)) { - // if version of both pub and cachePub are not equal, or sourceAddress of both are not equal, update - // eg: sessionserver crash, client reconnect to other sessionserver, sourceAddress changed, version not changed - // eg: client restart, sourceAddress and version are both changed - if (CLIENT_PUB_MAP.containsKey(cachePubAddr)) { - CLIENT_PUB_MAP.get(cachePubAddr).remove(registerId); - } - if (!CLIENT_PUB_MAP.containsKey(pubAddr)) { - CLIENT_PUB_MAP.putIfAbsent(pubAddr, new ConcurrentHashMap<>()); - } - CLIENT_PUB_MAP.get(pubAddr).put(registerId, pub); - isChanged = true; - } - } - } - } - Long lastVersion = cacheDatum.getVersion(); - if (isChanged) { - cacheDatum.setVersion(datum.getVersion()); - } - return new MergeResult(lastVersion, isChanged); + public Datum putSnapshot(String dataInfoId, Map toBeDeletedPubMap, + Map snapshotPubMap) { + return localDatumStorage.putSnapshot(dataInfoId, toBeDeletedPubMap, snapshotPubMap); } /** + * Getter method for property OWN_CONNECT_ID_INDEX. * - * @param datum - * @return + * @return property value of OWN_CONNECT_ID_INDEX */ - private static Long coverDatum(Datum datum) { - String dataCenter = datum.getDataCenter(); - String dataInfoId = datum.getDataInfoId(); - Datum cacheDatum = DATUM_MAP.get(dataCenter).get(dataInfoId); - if (datum.getVersion() != cacheDatum.getVersion()) { - DATUM_MAP.get(dataCenter).put(dataInfoId, datum); - Map pubMap = datum.getPubMap(); - Map cachePubMap = new HashMap<>(cacheDatum.getPubMap()); - for (Entry pubEntry : pubMap.entrySet()) { - String registerId = pubEntry.getKey(); - Publisher pub = pubEntry.getValue(); - String pubAddr = pub.getSourceAddress().getAddressString(); - if (!CLIENT_PUB_MAP.containsKey(pubAddr)) { - CLIENT_PUB_MAP.putIfAbsent(pubAddr, new ConcurrentHashMap<>()); - } - CLIENT_PUB_MAP.get(pubAddr).put(registerId, pub); - Publisher cachePub = cachePubMap.get(registerId); - if (cachePub != null - && pubAddr.equals(cachePub.getSourceAddress().getAddressString())) { - cachePubMap.remove(registerId); - } - } - if (!cachePubMap.isEmpty()) { - for (Publisher cachePub : cachePubMap.values()) { - CLIENT_PUB_MAP.get(cachePub.getSourceAddress().getAddressString()).remove( - cachePub.getRegisterId()); - } - } - } - return cacheDatum.getVersion(); + public Set getAllConnectIds() { + return localDatumStorage.getAllConnectIds(); } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumStorage.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumStorage.java new file mode 100644 index 000000000..fc71511c8 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/DatumStorage.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.cache; + +import java.util.Map; +import java.util.Set; + +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; + +/** + * + * @author kezhu.wukz + * @version $Id: DatumAccessService.java, v 0.1 2019-12-05 11:51 kezhu.wukz Exp $ + */ +public interface DatumStorage { + + /** + * get datum by specific dataCenter and dataInfoId + * + * @param dataCenter + * @param dataInfoId + * @return + */ + Datum get(String dataCenter, String dataInfoId); + + /** + * get datum of all datercenters by dataInfoId + * + * @param dataInfoId + * @return + */ + Map get(String dataInfoId); + + /** + * get all datum + * + * @return + */ + Map> getAll(); + + /** + * + * + * @param connectId + * @return + */ + Map getByConnectId(String connectId); + + /** + * get own publishers by connectId + */ + Map getOwnByConnectId(String connectId); + + /** + * Getter method for property OWN_CONNECT_ID_INDEX. + * + * @return property value of OWN_CONNECT_ID_INDEX + */ + Set getAllConnectIds(); + + /** + * put datum into cache + * + * @param changeType + * @param datum + * @return the last version before datum changed, if datum is not exist, return null + */ + MergeResult putDatum(DataChangeTypeEnum changeType, Datum datum); + + /** + * remove datum ant contains all pub data,and clean all the client map reference + * @param dataCenter + * @param dataInfoId + * @return + */ + boolean cleanDatum(String dataCenter, String dataInfoId); + + /** + * cover datum by snapshot + */ + Datum putSnapshot(String dataInfoId, Map toBeDeletedPubMap, + Map snapshotPubMap); + + Map getVersions(String dataInfoId); + +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/LocalDatumStorage.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/LocalDatumStorage.java new file mode 100644 index 000000000..fd1af059f --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/LocalDatumStorage.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.cache; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; +import com.alipay.sofa.registry.server.data.node.DataServerNode; +import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; + +/** + * datum storage of local dataCenter + * + * @author kezhu.wukz + * @version $Id: LocalDatumAccessService.java, v 0.1 2019-12-06 15:22 kezhu.wukz Exp $ + */ +public class LocalDatumStorage implements DatumStorage { + + public static final long ERROR_DATUM_VERSION = -2L; + + /** + * row: dataCenter + * column: dataInfoId + * value: datum + */ + protected final Map> DATUM_MAP = new ConcurrentHashMap<>(); + + /** + * all datum index + * + * row: ip:port + * column: registerId + * value: publisher + */ + protected final Map> ALL_CONNECT_ID_INDEX = new ConcurrentHashMap<>(); + + @Autowired + private DataServerConfig dataServerConfig; + + /** + * get datum by specific dataCenter and dataInfoId + * + * @param dataCenter + * @param dataInfoId + * @return + */ + public Datum get(String dataCenter, String dataInfoId) { + Map map = DATUM_MAP.get(dataCenter); + if (map != null) { + return map.get(dataInfoId); + } + return null; + } + + /** + * get datum of all datercenters by dataInfoId + * + * @param dataInfoId + * @return + */ + public Map get(String dataInfoId) { + Map datumMap = new HashMap<>(); + DATUM_MAP.forEach((dataCenter, datums) -> { + Datum datum = datums.get(dataInfoId); + if (datum != null) { + datumMap.put(dataCenter, datum); + } + }); + + return datumMap; + } + + /** + * get all datum + * + * @return + */ + public Map> getAll() { + return DATUM_MAP; + } + + /** + * + * + * @param connectId + * @return + */ + public Map getByConnectId(String connectId) { + return ALL_CONNECT_ID_INDEX.getOrDefault(connectId, null); + } + + /** + * get own publishers by connectId + */ + public Map getOwnByConnectId(String connectId) { + Map ownPubMap = new HashMap<>(); + Map allPubMap = ALL_CONNECT_ID_INDEX.getOrDefault(connectId, null); + if (allPubMap != null) { + for (Entry entry : allPubMap.entrySet()) { + String registerId = entry.getKey(); + Publisher publisher = entry.getValue(); + if (isOwnByMyself(publisher.getDataInfoId())) { + ownPubMap.put(registerId, publisher); + } + } + } + return ownPubMap; + } + + /** + * whether dataInfoId own by self + */ + protected boolean isOwnByMyself(String dataInfoId) { + DataServerNode dataServerNode = DataServerNodeFactory.computeDataServerNode( + dataServerConfig.getLocalDataCenter(), dataInfoId); + return DataServerConfig.IP.equals(dataServerNode.getIp()); + } + + /** + * put datum into cache + * + * @param changeType + * @param datum + * @return the last version before datum changed, if datum is not exist, return null + */ + public MergeResult putDatum(DataChangeTypeEnum changeType, Datum datum) { + MergeResult mergeResult; + String dataCenter = datum.getDataCenter(); + String dataInfoId = datum.getDataInfoId(); + Map map = getDatumMapByDataCenter(dataCenter); + + //first put UnPublisher datum(dataId group instanceId is null),can not add to cache + if (datum.getDataId() == null && map.get(dataInfoId) == null) { + mergeResult = new MergeResult(ERROR_DATUM_VERSION, false); + return mergeResult; + } + + // filter out the unPubs of datum when first put. + // Otherwise, "syncData" or "fetchData" when get Datum with unPubs, which will result something error + boolean[] exists = { true }; + Datum cacheDatum = map.computeIfAbsent(dataInfoId, k -> filterUnPubs(exists, datum)); + if (!exists[0]) { + Iterator> iterator = datum.getPubMap().entrySet().iterator(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + Publisher publisher = entry.getValue(); + addToIndex(publisher); + } + mergeResult = new MergeResult(null, true); + } else { + if (changeType == DataChangeTypeEnum.MERGE) { + mergeResult = mergeDatum(cacheDatum, datum); + } else { + Long lastVersion = coverDatum(datum); + mergeResult = new MergeResult(lastVersion, true); + } + } + return mergeResult; + } + + /** + * remove unPubs from datum + */ + private Datum filterUnPubs(boolean[] exists, Datum datum) { + Iterator> iterator = datum.getPubMap().entrySet().iterator(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + Publisher publisher = entry.getValue(); + if (publisher instanceof UnPublisher) { + //first put to cache,UnPublisher data must remove,not so got error pub data exist + iterator.remove(); + } + } + exists[0] = false; + return datum; + } + + private Map getDatumMapByDataCenter(String dataCenter) { + Map map = DATUM_MAP.get(dataCenter); + if (map == null) { + map = new ConcurrentHashMap<>(); + Map ret = DATUM_MAP.putIfAbsent(dataCenter, map); + if (ret != null) { + map = ret; + } + } + return map; + } + + /** + * remove datum ant contains all pub data,and clean all the client map reference + * @param dataCenter + * @param dataInfoId + * @return + */ + public boolean cleanDatum(String dataCenter, String dataInfoId) { + + Map datumMap = DATUM_MAP.get(dataCenter); + if (datumMap != null) { + Datum cacheDatum = datumMap.remove(dataInfoId); + if (cacheDatum != null) { + Map cachePubMap = cacheDatum.getPubMap(); + + for (Entry cachePubEntry : cachePubMap.entrySet()) { + String registerId = cachePubEntry.getKey(); + Publisher cachePub = cachePubEntry.getValue(); + //remove from cache + if (cachePub != null) { + cachePubMap.remove(registerId); + removeFromIndex(cachePub); + } + } + return true; + } + } + return false; + } + + /** + * merge datum in cache + * + * @param datum + * @return + */ + private MergeResult mergeDatum(Datum cacheDatum, Datum datum) { + boolean isChanged = false; + Map cachePubMap = cacheDatum.getPubMap(); + Map pubMap = datum.getPubMap(); + for (Entry pubEntry : pubMap.entrySet()) { + String registerId = pubEntry.getKey(); + Publisher pub = pubEntry.getValue(); + Publisher cachePub = cachePubMap.get(registerId); + if (mergePublisher(pub, cachePubMap, cachePub)) { + isChanged = true; + } + } + Long lastVersion = cacheDatum.getVersion(); + if (isChanged) { + cacheDatum.setVersion(datum.getVersion()); + } + return new MergeResult(lastVersion, isChanged); + } + + /** + * cover datum by snapshot + */ + public Datum putSnapshot(String dataInfoId, Map toBeDeletedPubMap, + Map snapshotPubMap) { + // get cache datum + Map datumMap = getDatumMapByDataCenter(dataServerConfig.getLocalDataCenter()); + Datum cacheDatum = datumMap.get(dataInfoId); + if (cacheDatum == null) { + cacheDatum = new Datum(dataInfoId, dataServerConfig.getLocalDataCenter()); + Publisher publisher = snapshotPubMap.values().iterator().next(); + cacheDatum.setInstanceId(publisher.getInstanceId()); + cacheDatum.setDataId(publisher.getDataId()); + cacheDatum.setGroup(publisher.getGroup()); + Datum datum = datumMap.putIfAbsent(dataInfoId, cacheDatum); + if (datum != null) { + cacheDatum = datum; + } + } + //remove toBeDeletedPubMap from cacheDatum + for (Entry toBeDeletedPubEntry : toBeDeletedPubMap.entrySet()) { + String registerId = toBeDeletedPubEntry.getKey(); + Publisher toBeDeletedPub = toBeDeletedPubEntry.getValue(); + if (cacheDatum != null) { + cacheDatum.getPubMap().remove(registerId); + removeFromIndex(toBeDeletedPub); + } + } + // add snapshotPubMap to cacheDatum + for (Entry pubEntry : snapshotPubMap.entrySet()) { + String registerId = pubEntry.getKey(); + Publisher snapshotPub = pubEntry.getValue(); + Publisher cachePub = cacheDatum.getPubMap().put(registerId, snapshotPub); + if (cachePub != null) { + removeFromIndex(cachePub); + } + addToIndex(snapshotPub); + } + + cacheDatum.updateVersion(); + + return cacheDatum; + } + + @Override + public Map getVersions(String dataInfoId) { + Map versions = new HashMap<>(1); + Map datumMap = this.get(dataInfoId); + if (datumMap != null) { + for (Map.Entry entry : datumMap.entrySet()) { + String dataCenter = entry.getKey(); + Datum datum = entry.getValue(); + versions.put(dataCenter, datum.getVersion()); + } + } + return versions; + } + + private boolean mergePublisher(Publisher pub, Map cachePubMap, + Publisher cachePub) { + boolean isChanged = false; + String registerId = pub.getRegisterId(); + if (pub instanceof UnPublisher) { + //remove from cache + if (cachePub != null && pub.getRegisterTimestamp() > cachePub.getRegisterTimestamp()) { + cachePubMap.remove(registerId); + removeFromIndex(cachePub); + isChanged = true; + } + } else { + long version = pub.getVersion(); + long cacheVersion = cachePub == null ? 0L : cachePub.getVersion(); + if (cacheVersion <= version) { + cachePubMap.put(registerId, pub); + // connectId and cacheConnectId may not be equal, so indexes need to be deleted and added, rather than overwritten directly. + // why connectId and cacheConnectId may not be equal? + // eg: sessionserver crash, client(RegistryClient but not ConfregClient) reconnect to other sessionserver, sourceAddress changed, version not changed + removeFromIndex(cachePub); + addToIndex(pub); + isChanged = true; + } + } + return isChanged; + } + + /** + * + * @param datum + * @return + */ + private Long coverDatum(Datum datum) { + String dataCenter = datum.getDataCenter(); + String dataInfoId = datum.getDataInfoId(); + Datum cacheDatum = DATUM_MAP.get(dataCenter).get(dataInfoId); + if (datum.getVersion() != cacheDatum.getVersion()) { + DATUM_MAP.get(dataCenter).put(dataInfoId, datum); + Map pubMap = datum.getPubMap(); + Map cachePubMap = new HashMap<>(cacheDatum.getPubMap()); + for (Entry pubEntry : pubMap.entrySet()) { + String registerId = pubEntry.getKey(); + Publisher pub = pubEntry.getValue(); + addToIndex(pub); + Publisher cachePub = cachePubMap.get(registerId); + if (cachePub != null && getConnectId(pub).equals(getConnectId(cachePub))) { + cachePubMap.remove(registerId); + } + } + if (!cachePubMap.isEmpty()) { + for (Publisher cachePub : cachePubMap.values()) { + removeFromIndex(cachePub); + } + } + } + return cacheDatum.getVersion(); + } + + private void removeFromIndex(Publisher publisher) { + if (publisher == null) { + return; + } + String connectId = getConnectId(publisher); + + // remove from ALL_CONNECT_ID_INDEX + Map publisherMap = ALL_CONNECT_ID_INDEX.get(connectId); + if (publisherMap != null) { + publisherMap.remove(publisher.getRegisterId()); + } + } + + private void addToIndex(Publisher publisher) { + if (publisher == null) { + return; + } + String connectId = getConnectId(publisher); + + // add to ALL_CONNECT_ID_INDEX + Map publisherMap = ALL_CONNECT_ID_INDEX + .computeIfAbsent(connectId, s -> new ConcurrentHashMap<>()); + publisherMap.put(publisher.getRegisterId(), publisher); + + } + + private String getConnectId(Publisher cachePub) { + return WordCache.getInstance().getWordCache( + cachePub.getSourceAddress().getAddressString() + ValueConstants.CONNECT_ID_SPLIT + + cachePub.getTargetAddress().getAddressString()); + } + + /** + * Getter method for property OWN_CONNECT_ID_INDEX. + * + * @return property value of OWN_CONNECT_ID_INDEX + */ + public Set getAllConnectIds() { + return ALL_CONNECT_ID_INDEX.keySet(); + } + +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/UnPublisher.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/UnPublisher.java index 5db5d80b9..1f3488dc8 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/UnPublisher.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/cache/UnPublisher.java @@ -45,6 +45,6 @@ public UnPublisher(String dataInfoId, String registerId, long registerTimeStamp) @Override public DataType getDataType() { - return DataType.UNPUBLISHER; + return DataType.UN_PUBLISHER; } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/ChangeData.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/ChangeData.java index 166f8fa48..f3e541877 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/ChangeData.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/ChangeData.java @@ -16,11 +16,11 @@ */ package com.alipay.sofa.registry.server.data.change; -import com.alipay.sofa.registry.common.model.dataserver.Datum; - import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import com.alipay.sofa.registry.common.model.dataserver.Datum; + /** * changed data * diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataChangeHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataChangeHandler.java index 6bf66a946..f79dc8682 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataChangeHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataChangeHandler.java @@ -16,21 +16,33 @@ */ package com.alipay.sofa.registry.server.data.change; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.cache.LocalDatumStorage; import com.alipay.sofa.registry.server.data.cache.MergeResult; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventQueue; import com.alipay.sofa.registry.server.data.change.notify.IDataChangeNotifier; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; -import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; +import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; +import java.util.Map; import java.util.concurrent.Executor; /** @@ -39,40 +51,35 @@ * @author qian.lqlq * @version $Id: DataChangeHandler.java, v 0.1 2017-12-07 18:44 qian.lqlq Exp $ */ -public class DataChangeHandler implements InitializingBean { +public class DataChangeHandler { + + private static final Logger LOGGER = LoggerFactory + .getLogger(DataChangeHandler.class); - private static final Logger LOGGER = LoggerFactory.getLogger(DataChangeHandler.class); + private static final Logger LOGGER_START = LoggerFactory.getLogger("DATA-START-LOGS"); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private DataChangeEventCenter dataChangeEventCenter; + @Autowired + private DatumCache datumCache; + @Resource private List dataChangeNotifiers; - @Override - public void afterPropertiesSet() { - //init DataChangeEventCenter - dataChangeEventCenter.init(dataServerBootstrapConfig); - start(); - } - - /** - * - */ + @PostConstruct public void start() { DataChangeEventQueue[] queues = dataChangeEventCenter.getQueues(); int queueCount = queues.length; - Executor executor = ExecutorFactory.newFixedThreadPool(queueCount, - DataChangeHandler.class.getSimpleName()); - Executor notifyExecutor = ExecutorFactory.newFixedThreadPool( - dataServerBootstrapConfig.getQueueCount() * 5, this.getClass().getSimpleName()); + Executor executor = ExecutorFactory.newFixedThreadPool(queueCount, DataChangeHandler.class.getSimpleName()); + Executor notifyExecutor = ExecutorFactory + .newFixedThreadPool(dataServerConfig.getQueueCount() * 5, this.getClass().getSimpleName()); for (int idx = 0; idx < queueCount; idx++) { final DataChangeEventQueue dataChangeEventQueue = queues[idx]; final String name = dataChangeEventQueue.getName(); - LOGGER.info("[DataChangeHandler] begin to notify datum in queue:{}", name); executor.execute(() -> { while (true) { try { @@ -83,7 +90,7 @@ public void start() { } } }); - LOGGER.info("[DataChangeHandler] notify datum in queue:{} success", name); + LOGGER_START.info("[DataChangeHandler] notify datum in queue:{} success", name); } } @@ -108,66 +115,85 @@ public ChangeNotifier(ChangeData changeData, String name) { @Override public void run() { - Datum datum = changeData.getDatum(); - String dataCenter = datum.getDataCenter(); - String dataInfoId = datum.getDataInfoId(); - long version = datum.getVersion(); - DataSourceTypeEnum sourceType = changeData.getSourceType(); - DataChangeTypeEnum changeType = changeData.getChangeType(); - try { - if (sourceType == DataSourceTypeEnum.CLEAN) { - if (DatumCache.cleanDatum(dataCenter, dataInfoId)) { - LOGGER - .info( - "[DataChangeHandler][{}] clean datum, dataCenter={}, dataInfoId={}, version={},sourceType={}, changeType={}", - name, dataCenter, dataInfoId, version, sourceType, changeType); - } + if (changeData instanceof SnapshotData) { + SnapshotData snapshotData = (SnapshotData) changeData; + String dataInfoId = snapshotData.getDataInfoId(); + Map toBeDeletedPubMap = snapshotData.getToBeDeletedPubMap(); + Map snapshotPubMap = snapshotData.getSnapshotPubMap(); + Datum oldDatum = datumCache.get(dataServerConfig.getLocalDataCenter(), dataInfoId); + long lastVersion = oldDatum != null ? oldDatum.getVersion() : 0l; + Datum datum = datumCache.putSnapshot(dataInfoId, toBeDeletedPubMap, snapshotPubMap); + long version = datum != null ? datum.getVersion() : 0l; + LOGGER + .info( + "[DataChangeHandler][{}] snapshot handle,dataInfoId={}, version={}, lastVersion={}", + name, dataInfoId, version, lastVersion); + notify(datum, changeData.getSourceType(), null); + + } else { + Datum datum = changeData.getDatum(); + + String dataCenter = datum.getDataCenter(); + String dataInfoId = datum.getDataInfoId(); + DataSourceTypeEnum sourceType = changeData.getSourceType(); + DataChangeTypeEnum changeType = changeData.getChangeType(); + + if (changeType == DataChangeTypeEnum.MERGE + && sourceType != DataSourceTypeEnum.BACKUP + && sourceType != DataSourceTypeEnum.SYNC) { + //update version for pub or unPub merge to cache + //if the version product before merge to cache,it may be cause small version override big one + datum.updateVersion(); + } - } else { - Long lastVersion = null; + long version = datum.getVersion(); - if (sourceType == DataSourceTypeEnum.PUB_TEMP) { + try { + if (sourceType == DataSourceTypeEnum.CLEAN) { + if (datumCache.cleanDatum(dataCenter, dataInfoId)) { + LOGGER + .info( + "[DataChangeHandler][{}] clean datum, dataCenter={}, dataInfoId={}, version={},sourceType={}, changeType={}", + name, dataCenter, dataInfoId, version, sourceType, changeType); + } + + } else if (sourceType == DataSourceTypeEnum.PUB_TEMP) { notifyTempPub(datum, sourceType, changeType); - return; - } - MergeResult mergeResult = DatumCache.putDatum(changeType, datum); - lastVersion = mergeResult.getLastVersion(); + } else { + MergeResult mergeResult = datumCache.putDatum(changeType, datum); + Long lastVersion = mergeResult.getLastVersion(); + + if (lastVersion != null + && lastVersion.longValue() == LocalDatumStorage.ERROR_DATUM_VERSION) { + LOGGER + .error( + "[DataChangeHandler][{}] first put unPub datum into cache error, dataCenter={}, dataInfoId={}, version={}, sourceType={},isContainsUnPub={}", + name, dataCenter, dataInfoId, version, sourceType, + datum.isContainsUnPub()); + return; + } - if (lastVersion != null - && lastVersion.longValue() == DatumCache.ERROR_DATUM_VERSION) { LOGGER - .error( - "[DataChangeHandler][{}] first put unPub datum into cache error, dataCenter={}, dataInfoId={}, version={}, sourceType={},isContainsUnPub={}", - name, dataCenter, dataInfoId, version, sourceType, + .info( + "[DataChangeHandler][{}] datum handle,datum={},dataCenter={}, dataInfoId={}, version={}, lastVersion={}, sourceType={}, changeType={},changeFlag={},isContainsUnPub={}", + name, datum.hashCode(), dataCenter, dataInfoId, version, + lastVersion, sourceType, changeType, mergeResult.isChangeFlag(), datum.isContainsUnPub()); - return; - } - - boolean changeFlag = mergeResult.isChangeFlag(); - - LOGGER - .info( - "[DataChangeHandler][{}] datum handle,datum={},dataCenter={}, dataInfoId={}, version={}, lastVersion={}, sourceType={}, changeType={},changeFlag={},isContainsUnPub={}", - name, datum.hashCode(), dataCenter, dataInfoId, version, lastVersion, - sourceType, changeType, changeFlag, datum.isContainsUnPub()); - //lastVersion null means first add datum - if (lastVersion == null || version != lastVersion) { - if (changeFlag) { - for (IDataChangeNotifier notifier : dataChangeNotifiers) { - if (notifier.getSuitableSource().contains(sourceType)) { - notifier.notify(datum, lastVersion); - } + //lastVersion null means first add datum + if (lastVersion == null || version != lastVersion) { + if (mergeResult.isChangeFlag()) { + notify(datum, sourceType, lastVersion); } } } + } catch (Exception e) { + LOGGER + .error( + "[DataChangeHandler][{}] put datum into cache error, dataCenter={}, dataInfoId={}, version={}, sourceType={},isContainsUnPub={}", + name, dataCenter, dataInfoId, version, sourceType, + datum.isContainsUnPub(), e); } - } catch (Exception e) { - LOGGER - .error( - "[DataChangeHandler][{}] put datum into cache error, dataCenter={}, dataInfoId={}, version={}, sourceType={},isContainsUnPub={}", - name, dataCenter, dataInfoId, version, sourceType, datum.isContainsUnPub(), - e); } } @@ -178,17 +204,30 @@ private void notifyTempPub(Datum datum, DataSourceTypeEnum sourceType, String dataCenter = datum.getDataCenter(); String dataInfoId = datum.getDataInfoId(); long version = datum.getVersion(); + + Datum existDatum = datumCache.get(dataCenter, dataInfoId); + if (existDatum != null) { + Map cachePubMap = existDatum.getPubMap(); + if (cachePubMap != null && !cachePubMap.isEmpty()) { + datum.getPubMap().putAll(cachePubMap); + } + } + LOGGER .info( - "[DataChangeHandler][{}] datum handle temp pub,datum={},dataCenter={}, dataInfoId={}, version={}, sourceType={}, changeType={},isContainsUnPub={}", - name, datum.hashCode(), dataCenter, dataInfoId, version, sourceType, - changeType, datum.isContainsUnPub()); + "[DataChangeHandler][{}] datum handle temp pub,datum={},dataCenter={}, dataInfoId={}, version={}, sourceType={}, changeType={}", + name, datum.hashCode(), dataCenter, dataInfoId, version, sourceType, changeType); + notify(datum, sourceType, null); + } + + private void notify(Datum datum, DataSourceTypeEnum sourceType, Long lastVersion) { for (IDataChangeNotifier notifier : dataChangeNotifiers) { if (notifier.getSuitableSource().contains(sourceType)) { - notifier.notify(datum, null); + notifier.notify(datum, lastVersion); } } } } + } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataSourceTypeEnum.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataSourceTypeEnum.java index c41a14f51..b2787fbb9 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataSourceTypeEnum.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/DataSourceTypeEnum.java @@ -46,5 +46,10 @@ public enum DataSourceTypeEnum { /** * local dataInfo check,not belong this node schedule remove */ - CLEAN + CLEAN, + + /** + * Snapshot data, after renew finds data inconsistent + */ + SNAPSHOT, } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/SnapshotData.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/SnapshotData.java new file mode 100644 index 000000000..495019d65 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/SnapshotData.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.change; + +import java.util.Map; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + +import com.alipay.sofa.registry.common.model.store.Publisher; + +/** + * changed data + * + * @author kezhu.wukz + * @version $Id: ChangeData.java, v 0.1 2019-07-12 16:23 kezhu.wukz Exp $ + */ +public class SnapshotData extends ChangeData { + + private String dataInfoId; + + private Map toBeDeletedPubMap; + + private Map snapshotPubMap; + + public SnapshotData(String dataInfoId, Map toBeDeletedPubMap, + Map snapshotPubMap) { + super(null, 0, DataSourceTypeEnum.SNAPSHOT, null); + this.dataInfoId = dataInfoId; + this.toBeDeletedPubMap = toBeDeletedPubMap; + this.snapshotPubMap = snapshotPubMap; + } + + /** + * Getter method for property dataInfoId. + * + * @return property value of dataInfoId + */ + public String getDataInfoId() { + return dataInfoId; + } + + /** + * Getter method for property toBeDeletedPubMap. + * + * @return property value of toBeDeletedPubMap + */ + public Map getToBeDeletedPubMap() { + return toBeDeletedPubMap; + } + + /** + * Getter method for property snapshotPubMap. + * + * @return property value of snapshotPubMap + */ + public Map getSnapshotPubMap() { + return snapshotPubMap; + } + + @Override + public long getDelay(TimeUnit unit) { + return 0; + } + + @Override + public int compareTo(Delayed o) { + return -1; + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/ClientChangeEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/ClientChangeEvent.java index c063f50d4..b2eb9f036 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/ClientChangeEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/ClientChangeEvent.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.data.change.event; +import com.alipay.sofa.registry.util.DatumVersionUtil; + /** * * @author qian.lqlq @@ -41,7 +43,7 @@ public ClientChangeEvent(String host, String dataCenter, long occurredTimestamp) this.host = host; this.dataCenter = dataCenter; this.occurredTimestamp = occurredTimestamp; - this.version = System.currentTimeMillis(); + this.version = DatumVersionUtil.nextId(); } @Override diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventCenter.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventCenter.java index 58c5496e2..01b5e220d 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventCenter.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventCenter.java @@ -16,16 +16,21 @@ */ package com.alipay.sofa.registry.server.data.change.event; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.PublishType; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.cache.UnPublisher; import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; -import java.util.concurrent.atomic.AtomicBoolean; - /** * * @author qian.lqlq @@ -44,16 +49,20 @@ public class DataChangeEventCenter { */ private DataChangeEventQueue[] dataChangeEventQueues; - /** - * - * @param config - */ - public void init(DataServerConfig config) { + @Autowired + private DataServerConfig dataServerConfig; + + @Autowired + private DatumCache datumCache; + + @PostConstruct + public void init() { if (isInited.compareAndSet(false, true)) { - queueCount = config.getQueueCount(); + queueCount = dataServerConfig.getQueueCount(); dataChangeEventQueues = new DataChangeEventQueue[queueCount]; for (int idx = 0; idx < queueCount; idx++) { - dataChangeEventQueues[idx] = new DataChangeEventQueue(idx, config); + dataChangeEventQueues[idx] = new DataChangeEventQueue(idx, dataServerConfig, this, + datumCache); dataChangeEventQueues[idx].start(); } } @@ -101,6 +110,16 @@ public void onChange(ClientChangeEvent event) { } } + /** + * + * @param event + */ + public void onChange(DatumSnapshotEvent event) { + for (DataChangeEventQueue dataChangeEventQueue : dataChangeEventQueues) { + dataChangeEventQueue.onChange(event); + } + } + /** * * @param changeType @@ -118,7 +137,7 @@ public void sync(DataChangeTypeEnum changeType, DataSourceTypeEnum sourceType, D * @param key * @return */ - private int hash(String key) { + public int hash(String key) { if (queueCount > 1) { return Math.abs(key.hashCode() % queueCount); } else { diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventQueue.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventQueue.java index 45e5cffab..985c57702 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventQueue.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeEventQueue.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.server.data.change.event; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.locks.ReentrantLock; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.log.Logger; @@ -26,19 +36,12 @@ import com.alipay.sofa.registry.server.data.change.ChangeData; import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; +import com.alipay.sofa.registry.server.data.change.SnapshotData; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.node.DataServerNode; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; import com.google.common.collect.Interners; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.locks.ReentrantLock; - /** * a queue of DataChangeEvent * @@ -47,8 +50,16 @@ */ public class DataChangeEventQueue { - private static final Logger LOGGER = LoggerFactory - .getLogger(DataChangeEventQueue.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(DataChangeEventQueue.class); + + private static final Logger LOGGER_START = LoggerFactory + .getLogger("DATA-START-LOGS"); + + private static final Logger RENEW_LOGGER = LoggerFactory + .getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[DataChangeEventQueue]"); /** * @@ -63,27 +74,36 @@ public class DataChangeEventQueue { /** * */ - private final Map> CHANGE_DATA_MAP = new ConcurrentHashMap<>(); + private final Map> CHANGE_DATA_MAP_FOR_MERGE = new ConcurrentHashMap<>(); /** * */ - private final DelayQueue CHANGE_QUEUE = new DelayQueue(); + private final DelayQueue CHANGE_QUEUE = new DelayQueue(); private final int notifyIntervalMs; - private final ReentrantLock lock = new ReentrantLock(); + private final int notifyTempDataIntervalMs; + + private final ReentrantLock lock = new ReentrantLock(); + + private final int queueIdx; private DataServerConfig dataServerConfig; + private DataChangeEventCenter dataChangeEventCenter; + + private DatumCache datumCache; + /** * constructor - * @param idx + * @param queueIdx * @param dataServerConfig */ - public DataChangeEventQueue(int idx, DataServerConfig dataServerConfig) { - - this.name = String.format("%s_%s", DataChangeEventQueue.class.getSimpleName(), idx); + public DataChangeEventQueue(int queueIdx, DataServerConfig dataServerConfig, + DataChangeEventCenter dataChangeEventCenter, DatumCache datumCache) { + this.queueIdx = queueIdx; + this.name = String.format("%s_%s", DataChangeEventQueue.class.getSimpleName(), queueIdx); this.dataServerConfig = dataServerConfig; int queueSize = dataServerConfig.getQueueSize(); if (queueSize <= 0) { @@ -92,6 +112,9 @@ public DataChangeEventQueue(int idx, DataServerConfig dataServerConfig) { eventQueue = new LinkedBlockingDeque<>(queueSize); } this.notifyIntervalMs = dataServerConfig.getNotifyIntervalMs(); + this.notifyTempDataIntervalMs = dataServerConfig.getNotifyTempDataIntervalMs(); + this.dataChangeEventCenter = dataChangeEventCenter; + this.datumCache = datumCache; } /** @@ -100,7 +123,12 @@ public DataChangeEventQueue(int idx, DataServerConfig dataServerConfig) { * @param event */ public void onChange(IDataChangeEvent event) { - eventQueue.add(event); + try { + eventQueue.add(event); + } catch (Throwable e) { + LOGGER.error("Error onChange: " + e.getMessage(), e); + throw e; + } } /** @@ -120,14 +148,20 @@ public ChangeData take() throws InterruptedException { ChangeData changeData = CHANGE_QUEUE.take(); lock.lock(); try { - Datum datum = changeData.getDatum(); - CHANGE_DATA_MAP.get(datum.getDataCenter()).remove(datum.getDataInfoId()); + removeMapForMerge(changeData); return changeData; } finally { lock.unlock(); } } + private void removeMapForMerge(ChangeData changeData) { + Datum datum = changeData.getDatum(); + if (changeData.getSourceType() != DataSourceTypeEnum.PUB_TEMP && datum != null) { + CHANGE_DATA_MAP_FOR_MERGE.get(datum.getDataCenter()).remove(datum.getDataInfoId()); + } + } + /** * * @param dataCenter @@ -138,10 +172,10 @@ public ChangeData take() throws InterruptedException { */ private ChangeData getChangeData(String dataCenter, String dataInfoId, DataSourceTypeEnum sourceType, DataChangeTypeEnum changeType) { - Map map = CHANGE_DATA_MAP.get(dataCenter); + Map map = CHANGE_DATA_MAP_FOR_MERGE.get(dataCenter); if (map == null) { Map newMap = new ConcurrentHashMap<>(); - map = CHANGE_DATA_MAP.putIfAbsent(dataCenter, newMap); + map = CHANGE_DATA_MAP_FOR_MERGE.putIfAbsent(dataCenter, newMap); if (map == null) { map = newMap; } @@ -164,9 +198,8 @@ private ChangeData getChangeData(String dataCenter, String dataInfoId, * */ public void start() { - LOGGER.info("[{}] begin start DataChangeEventQueue", getName()); - Executor executor = ExecutorFactory.newSingleThreadExecutor( - String.format("%s_%s", DataChangeEventQueue.class.getSimpleName(), getName())); + Executor executor = ExecutorFactory + .newSingleThreadExecutor(String.format("%s_%s", DataChangeEventQueue.class.getSimpleName(), getName())); executor.execute(() -> { while (true) { try { @@ -174,26 +207,42 @@ public void start() { DataChangeScopeEnum scope = event.getScope(); if (scope == DataChangeScopeEnum.DATUM) { DataChangeEvent dataChangeEvent = (DataChangeEvent) event; - handleDatum(dataChangeEvent.getChangeType(), - dataChangeEvent.getSourceType(), dataChangeEvent.getDatum()); + //Temporary push data will be notify as soon as,and not merge to normal pub data; + if (dataChangeEvent.getSourceType() == DataSourceTypeEnum.PUB_TEMP) { + addTempChangeData(dataChangeEvent.getDatum(), dataChangeEvent.getChangeType(), + dataChangeEvent.getSourceType()); + } else { + handleDatum(dataChangeEvent.getChangeType(), dataChangeEvent.getSourceType(), + dataChangeEvent.getDatum()); + } } else if (scope == DataChangeScopeEnum.CLIENT) { - handleHost((ClientChangeEvent) event); + handleClientOff((ClientChangeEvent) event); + } else if (scope == DataChangeScopeEnum.SNAPSHOT) { + handleSnapshot((DatumSnapshotEvent) event); } } catch (Throwable e) { LOGGER.error("[{}] handle change event failed", getName(), e); } } }); - LOGGER.info("[{}] start DataChangeEventQueue success", getName()); + LOGGER_START.info("[{}] start DataChangeEventQueue success", getName()); } - private void handleHost(ClientChangeEvent event) { - String clientHost = event.getHost(); - synchronized (Interners.newWeakInterner().intern(clientHost)) { - Map pubMap = DatumCache.getByHost(clientHost); + private void handleClientOff(ClientChangeEvent event) { + String connectId = event.getHost(); + synchronized (Interners.newWeakInterner().intern(connectId)) { + Map pubMap = datumCache.getByConnectId(connectId); if (pubMap != null && !pubMap.isEmpty()) { + LOGGER.info( + "[{}] client off begin, connectId={}, occurTimestamp={}, all pubSize={}", + getName(), connectId, event.getOccurredTimestamp(), pubMap.size()); int count = 0; for (Publisher publisher : pubMap.values()) { + // Only care dataInfoIds which belong to this queue + if (!belongTo(publisher.getDataInfoId())) { + continue; + } + DataServerNode dataServerNode = DataServerNodeFactory.computeDataServerNode( dataServerConfig.getLocalDataCenter(), publisher.getDataInfoId()); //current dataCenter backup data need not unPub,it will be unPub by backup sync event @@ -208,11 +257,9 @@ private void handleHost(ClientChangeEvent event) { } LOGGER .info( - "[{}] client off handle, host={}, occurTimestamp={},version={},handle pub size={}", - getName(), clientHost, event.getOccurredTimestamp(), event.getVersion(), + "[{}] client off handle, connectId={}, occurTimestamp={}, version={}, handle pubSize={}", + getName(), connectId, event.getOccurredTimestamp(), event.getVersion(), count); - } else { - LOGGER.info("[{}] no datum to handle, host={}", getName(), clientHost); } } } @@ -243,7 +290,7 @@ private void handleDatum(DataChangeTypeEnum changeType, DataSourceTypeEnum sourc // and version of cachePub is greater than version of pub, should be ignored if (!(pub instanceof UnPublisher) && !(cachePub instanceof UnPublisher) && pub.getSourceAddress().equals(cachePub.getSourceAddress()) - && cachePub.getVersion() >= pub.getVersion()) { + && cachePub.getVersion() > pub.getVersion()) { continue; } } @@ -255,4 +302,77 @@ private void handleDatum(DataChangeTypeEnum changeType, DataSourceTypeEnum sourc lock.unlock(); } } + + private void handleSnapshot(DatumSnapshotEvent event) { + String connectId = event.getConnectId(); + Map cachePubMap = event.getCachePubMap(); + Map snapshotPubMap = event.getPubMap(); + + // build SnapshotData + Map dataInfoId2SnapshotData = new HashMap<>(); + synchronized (Interners.newWeakInterner().intern(connectId)) { + for (Map.Entry entry : snapshotPubMap.entrySet()) { + String registerId = entry.getKey(); + Publisher publisher = entry.getValue(); + String dataInfoId = publisher.getDataInfoId(); + + // Only care dataInfoIds which belong to this queue + if (!belongTo(dataInfoId)) { + continue; + } + + SnapshotData snapshotData = getOrCreateSnapshotData(dataInfoId2SnapshotData, + dataInfoId); + snapshotData.getSnapshotPubMap().put(registerId, publisher); + } + for (Map.Entry entry : cachePubMap.entrySet()) { + String registerId = entry.getKey(); + Publisher publisher = entry.getValue(); + String dataInfoId = publisher.getDataInfoId(); + + // Only care dataInfoIds which belong to this queue + if (!belongTo(dataInfoId)) { + continue; + } + + SnapshotData snapshotData = getOrCreateSnapshotData(dataInfoId2SnapshotData, + dataInfoId); + snapshotData.getToBeDeletedPubMap().put(registerId, publisher); + } + } + + // put all SnapshotDatas to queue + for (SnapshotData snapshotData : dataInfoId2SnapshotData.values()) { + RENEW_LOGGER + .info( + "SnapshotData: connectId={}, dataInfoId={}, cachePubSize={}, snapshotPubSize={}", + connectId, snapshotData.getDataInfoId(), snapshotData.getToBeDeletedPubMap() + .size(), snapshotData.getSnapshotPubMap().size()); + CHANGE_QUEUE.put(snapshotData); + } + } + + private SnapshotData getOrCreateSnapshotData(Map dataInfoId2SnapshotData, + String dataInfoId) { + SnapshotData snapshotData = dataInfoId2SnapshotData.get(dataInfoId); + if (snapshotData == null) { + snapshotData = new SnapshotData(dataInfoId, new HashMap<>(), new HashMap<>()); + dataInfoId2SnapshotData.put(dataInfoId, snapshotData); + } + return snapshotData; + } + + private void addTempChangeData(Datum targetDatum, DataChangeTypeEnum changeType, + DataSourceTypeEnum sourceType) { + ChangeData tempChangeData = new ChangeData(targetDatum, this.notifyTempDataIntervalMs, + sourceType, changeType); + CHANGE_QUEUE.put(tempChangeData); + } + + /** + * Determine whether dataInfoId belongs to the current queue + */ + private boolean belongTo(String dataInfoId) { + return this.queueIdx == this.dataChangeEventCenter.hash(dataInfoId); + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeScopeEnum.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeScopeEnum.java index bfba719e5..94bbdb0d3 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeScopeEnum.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DataChangeScopeEnum.java @@ -22,5 +22,5 @@ * @version $Id: DataChangeScopeEnum.java, v 0.1 2018-05-10 16:51 qian.lqlq Exp $ */ public enum DataChangeScopeEnum { - CLIENT, DATUM + CLIENT, DATUM, SNAPSHOT } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DatumSnapshotEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DatumSnapshotEvent.java new file mode 100644 index 000000000..af08c25c6 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/event/DatumSnapshotEvent.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.change.event; + +import java.util.Map; + +import com.alipay.sofa.registry.common.model.store.Publisher; + +/** + * + * @author kezhu.wukz + * @version $Id: DatumSnapshotEvent.java, v 0.1 2019-05-30 18:22 kezhu.wukz Exp $ + */ +public class DatumSnapshotEvent implements IDataChangeEvent { + + /** connId, format is ip:port */ + private String connectId; + + private Map pubMap; + + private Map cachePubMap; + + public DatumSnapshotEvent(String connectId, Map cachePubMap, + Map pubMap) { + this.connectId = connectId; + this.cachePubMap = cachePubMap; + this.pubMap = pubMap; + } + + /** + * Getter method for property connectId. + * + * @return property value of connectId + */ + public String getConnectId() { + return connectId; + } + + /** + * Getter method for property pubMap. + * + * @return property value of pubMap + */ + public Map getPubMap() { + return pubMap; + } + + /** + * Getter method for property cachePubMap. + * + * @return property value of cachePubMap + */ + public Map getCachePubMap() { + return cachePubMap; + } + + @Override + public DataChangeScopeEnum getScope() { + return DataChangeScopeEnum.SNAPSHOT; + } + +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/BackUpNotifier.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/BackUpNotifier.java index 293376031..45ac0e0b5 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/BackUpNotifier.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/BackUpNotifier.java @@ -16,14 +16,15 @@ */ package com.alipay.sofa.registry.server.data.change.notify; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; import com.alipay.sofa.registry.server.data.datasync.Operator; import com.alipay.sofa.registry.server.data.datasync.SyncDataService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashSet; -import java.util.Set; /** * @@ -39,7 +40,6 @@ public class BackUpNotifier implements IDataChangeNotifier { public Set getSuitableSource() { Set set = new HashSet<>(); set.add(DataSourceTypeEnum.PUB); - set.add(DataSourceTypeEnum.SYNC); return set; } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SessionServerNotifier.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SessionServerNotifier.java index 727156d35..f66fb747e 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SessionServerNotifier.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SessionServerNotifier.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.server.data.change.notify; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.dataserver.Datum; @@ -29,16 +39,11 @@ import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; +import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; /** * Notify session DataChangeRequest,if fail get result callback retry @@ -54,7 +59,7 @@ public class SessionServerNotifier implements IDataChangeNotifier { private AsyncHashedWheelTimer asyncHashedWheelTimer; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private Exchange boltExchange; @@ -62,13 +67,19 @@ public class SessionServerNotifier implements IDataChangeNotifier { @Autowired private SessionServerConnectionFactory sessionServerConnectionFactory; - public SessionServerNotifier() { + @Autowired + private DatumCache datumCache; + + @PostConstruct + public void init() { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setDaemon(true); asyncHashedWheelTimer = new AsyncHashedWheelTimer(threadFactoryBuilder.setNameFormat( - "Registry-SessionServerNotifier-WheelTimer").build(), 100, TimeUnit.MILLISECONDS, 1024, - threadFactoryBuilder.setNameFormat("Registry-SessionServerNotifier-WheelExecutor-%d") - .build(), new TaskFailedCallback() { + "Registry-SessionServerNotifier-WheelTimer").build(), 500, TimeUnit.MILLISECONDS, 1024, + dataServerConfig.getSessionServerNotifierRetryExecutorThreadSize(), + dataServerConfig.getSessionServerNotifierRetryExecutorQueueSize(), threadFactoryBuilder + .setNameFormat("Registry-SessionServerNotifier-WheelExecutor-%d").build(), + new TaskFailedCallback() { @Override public void executionRejected(Throwable e) { LOGGER.error("executionRejected: " + e.getMessage(), e); @@ -86,6 +97,7 @@ public Set getSuitableSource() { Set set = new HashSet<>(); set.add(DataSourceTypeEnum.PUB); set.add(DataSourceTypeEnum.SYNC); + set.add(DataSourceTypeEnum.SNAPSHOT); return set; } @@ -93,7 +105,7 @@ public Set getSuitableSource() { public void notify(Datum datum, Long lastVersion) { DataChangeRequest request = new DataChangeRequest(datum.getDataInfoId(), datum.getDataCenter(), datum.getVersion()); - List connections = sessionServerConnectionFactory.getConnections(); + List connections = sessionServerConnectionFactory.getSessionConnections(); for (Connection connection : connections) { doNotify(new NotifyCallback(connection, request)); } @@ -105,21 +117,19 @@ private void doNotify(NotifyCallback notifyCallback) { try { //check connection active if (!connection.isFine()) { - if (LOGGER.isInfoEnabled()) { - LOGGER - .info(String - .format( - "connection from sessionserver(%s) is not fine, so ignore notify, retryTimes=%s,request=%s", - connection.getRemoteAddress(), notifyCallback.retryTimes, request)); - } + LOGGER + .info(String + .format( + "connection from sessionServer(%s) is not fine, so ignore notify, retryTimes=%s,request=%s", + connection.getRemoteAddress(), notifyCallback.retryTimes, request)); return; } - Server sessionServer = boltExchange.getServer(dataServerBootstrapConfig.getPort()); + Server sessionServer = boltExchange.getServer(dataServerConfig.getPort()); sessionServer.sendCallback(sessionServer.getChannel(connection.getRemoteAddress()), - request, notifyCallback, dataServerBootstrapConfig.getRpcTimeout()); + request, notifyCallback, dataServerConfig.getRpcTimeout()); } catch (Exception e) { LOGGER.error(String.format( - "invokeWithCallback failed: sessionserver(%s),retryTimes=%s, request=%s", + "invokeWithCallback failed: sessionServer(%s),retryTimes=%s, request=%s", connection.getRemoteAddress(), notifyCallback.retryTimes, request), e); onFailed(notifyCallback); } @@ -129,26 +139,32 @@ private void doNotify(NotifyCallback notifyCallback) { * on failed, retry if necessary */ private void onFailed(NotifyCallback notifyCallback) { + DataChangeRequest request = notifyCallback.request; Connection connection = notifyCallback.connection; notifyCallback.retryTimes++; - if (notifyCallback.retryTimes <= dataServerBootstrapConfig.getNotifySessionRetryTimes()) { + //check version, if it's fall behind, stop retry + long _currentVersion = datumCache.get(request.getDataCenter(), request.getDataInfoId()).getVersion(); + if (request.getVersion() != _currentVersion) { + LOGGER.info(String.format( + "current version change %s, retry version is %s, stop before retry! retryTimes=%s, request=%s", + _currentVersion, request.getVersion(), notifyCallback.retryTimes, request)); + return; + } + + if (notifyCallback.retryTimes <= dataServerConfig.getNotifySessionRetryTimes()) { this.asyncHashedWheelTimer.newTimeout(timeout -> { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("retrying notify sessionserver(%s), retryTimes=%s, request=%s", - connection.getRemoteAddress(), notifyCallback.retryTimes, request)); - } + LOGGER.info(String.format("retrying notify sessionServer(%s), retryTimes=%s, request=%s", + connection.getRemoteAddress(), notifyCallback.retryTimes, request)); //check version, if it's fall behind, stop retry - long currentVersion = DatumCache.get(request.getDataCenter(), request.getDataInfoId()).getVersion(); + long currentVersion = datumCache.get(request.getDataCenter(), request.getDataInfoId()).getVersion(); if (request.getVersion() == currentVersion) { doNotify(notifyCallback); } else { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format( - "current version change %s, retry version is %s, stop retry! retryTimes=%s, request=%s", - currentVersion, request.getVersion(), notifyCallback.retryTimes, request)); - } + LOGGER.info(String.format( + "current version change %s, retry version is %s, stop retry! retryTimes=%s, request=%s", + currentVersion, request.getVersion(), notifyCallback.retryTimes, request)); } }, getDelayTimeForRetry(notifyCallback.retryTimes), TimeUnit.MILLISECONDS); } else { @@ -159,9 +175,9 @@ private void onFailed(NotifyCallback notifyCallback) { } private long getDelayTimeForRetry(int retryTimes) { - long initialSleepTime = TimeUnit.MILLISECONDS.toMillis(dataServerBootstrapConfig + long initialSleepTime = TimeUnit.MILLISECONDS.toMillis(dataServerConfig .getNotifySessionRetryFirstDelay()); - long increment = TimeUnit.MILLISECONDS.toMillis(dataServerBootstrapConfig + long increment = TimeUnit.MILLISECONDS.toMillis(dataServerConfig .getNotifySessionRetryIncrementDelay()); long result = initialSleepTime + (increment * (retryTimes - 1)); return result >= 0L ? result : 0L; @@ -185,7 +201,7 @@ public void onCallback(Channel channel, Object message) { LOGGER .error(String .format( - "response not success when notify sessionserver(%s), retryTimes=%s, request=%s, response=%s", + "response not success when notify sessionServer(%s), retryTimes=%s, request=%s, response=%s", connection.getRemoteAddress(), retryTimes, request, result)); onFailed(this); } @@ -194,11 +210,16 @@ public void onCallback(Channel channel, Object message) { @Override public void onException(Channel channel, Throwable e) { LOGGER.error(String.format( - "exception when notify sessionserver(%s), retryTimes=%s, request=%s", + "exception when notify sessionServer(%s), retryTimes=%s, request=%s", connection.getRemoteAddress(), retryTimes, request), e); onFailed(this); } + @Override + public Executor getExecutor() { + return ExecutorFactory.NOTIFY_SESSION_CALLBACK_EXECUTOR; + } + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SnapshotBackUpNotifier.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SnapshotBackUpNotifier.java new file mode 100644 index 000000000..aab640fb2 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/SnapshotBackUpNotifier.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.change.notify; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; +import com.alipay.sofa.registry.server.data.datasync.SnapshotOperator; +import com.alipay.sofa.registry.server.data.datasync.SyncDataService; + +/** + * + * @author kezhu.wukz + * @version $Id: SnapshotBackUpNotifier.java, v 0.1 2019-07-12 18:40 kezhu.wukz Exp $ + */ +public class SnapshotBackUpNotifier implements IDataChangeNotifier { + + @Autowired + private SyncDataService syncDataService; + + @Override + public Set getSuitableSource() { + Set set = new HashSet<>(); + set.add(DataSourceTypeEnum.SNAPSHOT); + return set; + } + + @Override + public void notify(Datum datum, Long lastVersion) { + syncDataService.appendOperator(new SnapshotOperator(datum.getVersion(), lastVersion, datum, + DataSourceTypeEnum.BACKUP)); + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/TempPublisherNotifier.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/TempPublisherNotifier.java index b37fa7541..38096aa96 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/TempPublisherNotifier.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/change/notify/TempPublisherNotifier.java @@ -16,6 +16,13 @@ */ package com.alipay.sofa.registry.server.data.change.notify; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executor; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.remoting.InvokeCallback; import com.alipay.sofa.registry.common.model.CommonResponse; @@ -31,12 +38,6 @@ import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Executor; /** * @@ -54,7 +55,7 @@ public class TempPublisherNotifier implements IDataChangeNotifier { .getSimpleName()); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private Exchange boltExchange; @@ -72,7 +73,7 @@ public Set getSuitableSource() { @Override public void notify(Datum datum, Long lastVersion) { DataPushRequest request = new DataPushRequest(datum); - List connections = sessionServerConnectionFactory.getConnections(); + List connections = sessionServerConnectionFactory.getSessionConnections(); for (Connection connection : connections) { doNotify(new NotifyPushDataCallback(connection, request)); } @@ -83,7 +84,7 @@ private void doNotify(NotifyPushDataCallback notifyPushdataCallback) { Connection connection = notifyPushdataCallback.getConnection(); DataPushRequest request = notifyPushdataCallback.getRequest(); try { - Server sessionServer = boltExchange.getServer(dataServerBootstrapConfig.getPort()); + Server sessionServer = boltExchange.getServer(dataServerConfig.getPort()); sessionServer.sendCallback(sessionServer.getChannel(connection.getRemoteAddress()), request, new CallbackHandler() { @@ -96,7 +97,12 @@ public void onCallback(Channel channel, Object message) { public void onException(Channel channel, Throwable exception) { notifyPushdataCallback.onException(exception); } - }, dataServerBootstrapConfig.getRpcTimeout()); + + @Override + public Executor getExecutor() { + return notifyPushdataCallback.getExecutor(); + } + }, dataServerConfig.getRpcTimeout()); } catch (Exception e) { LOGGER.error("[TempPublisherNotifier] notify sessionserver {} failed, {}", connection.getRemoteIP(), request, e); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/SnapshotOperator.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/SnapshotOperator.java new file mode 100644 index 000000000..dd7aaf163 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/SnapshotOperator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.datasync; + +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; + +/** + * + * @author kezhu.wukz + * @version $Id: SnapshotOperator.java, v 0.1 2019-07-12 20:33 kezhu.wukz Exp $ + */ +public class SnapshotOperator extends Operator { + + public SnapshotOperator(Long version, Long sourceVersion, Datum datum, + DataSourceTypeEnum sourceType) { + super(version, sourceVersion, datum, sourceType); + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/AbstractAcceptorStore.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/AbstractAcceptorStore.java index c85ff165c..0c010c8f9 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/AbstractAcceptorStore.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/AbstractAcceptorStore.java @@ -16,6 +16,13 @@ */ package com.alipay.sofa.registry.server.data.datasync.sync; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.DelayQueue; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.NotifyDataSyncRequest; @@ -26,17 +33,14 @@ import com.alipay.sofa.registry.remoting.Server; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.datasync.AcceptorStore; import com.alipay.sofa.registry.server.data.datasync.Operator; +import com.alipay.sofa.registry.server.data.datasync.SnapshotOperator; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerConnectionFactory; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import com.alipay.sofa.registry.server.data.util.DelayItem; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.DelayQueue; +import com.alipay.sofa.registry.server.data.util.TimeUtil; /** * @@ -50,8 +54,6 @@ public abstract class AbstractAcceptorStore implements AcceptorStore { "[SyncDataService]"); private static final int DEFAULT_MAX_BUFFER_SIZE = 30; - private static final int DEFAULT_DELAY_TIMEOUT = 3000; - private static final int NOTIFY_RETRY = 3; @Autowired protected IMetaServerService metaServerService; @@ -60,11 +62,14 @@ public abstract class AbstractAcceptorStore implements AcceptorStore { private Exchange boltExchange; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private DataServerConnectionFactory dataServerConnectionFactory; + @Autowired + private DatumCache datumCache; + private Map> acceptors = new ConcurrentHashMap<>(); private Map> notifyAcceptorsCache = new ConcurrentHashMap<>(); @@ -104,13 +109,21 @@ public void addOperator(Operator operator) { Acceptor existAcceptor = acceptorMap.get(dataInfoId); if (existAcceptor == null) { - Acceptor newAcceptor = new Acceptor(DEFAULT_MAX_BUFFER_SIZE, dataInfoId, dataCenter); + Acceptor newAcceptor = new Acceptor(DEFAULT_MAX_BUFFER_SIZE, dataInfoId, + dataCenter, datumCache); existAcceptor = acceptorMap.putIfAbsent(dataInfoId, newAcceptor); if (existAcceptor == null) { existAcceptor = newAcceptor; } } - existAcceptor.appendOperator(operator); + + if (operator instanceof SnapshotOperator) { + //snapshot: clear the queue, Make other data retrieve the latest memory data + existAcceptor.clearBefore(); + } else { + existAcceptor.appendOperator(operator); + } + //put cache putCache(existAcceptor); } catch (Exception e) { @@ -163,7 +176,7 @@ private void removeCache(Acceptor acceptor) { } private void addQueue(Acceptor acceptor) { - delayQueue.put(new DelayItem(acceptor, DEFAULT_DELAY_TIMEOUT)); + delayQueue.put(new DelayItem(acceptor, dataServerConfig.getDataSyncDelayTimeout())); } private void notifyChange(Acceptor acceptor) { @@ -191,25 +204,31 @@ private void notifyChange(Acceptor acceptor) { continue; } - Connection connection = dataServerConnectionFactory.getConnection(targetDataIp); - if (connection == null) { - LOGGER.error(getLogByClass(String.format( - "Can not get notify data server connection!ip: %s", targetDataIp))); - continue; - } - LOGGER.info(getLogByClass("Notify data server {} change data {} to sync"), - connection.getRemoteIP(), request); - for (int tryCount = 0; tryCount < NOTIFY_RETRY; tryCount++) { + Server syncServer = boltExchange.getServer(dataServerConfig.getSyncDataPort()); + + for (int tryCount = 0; tryCount < dataServerConfig.getDataSyncNotifyRetry(); tryCount++) { try { - Server syncServer = boltExchange.getServer(dataServerBootstrapConfig - .getSyncDataPort()); + + Connection connection = dataServerConnectionFactory.getConnection(targetDataIp); + if (connection == null) { + LOGGER.error(getLogByClass(String.format( + "Can not get notify data server connection!ip: %s,retry=%s", + targetDataIp, tryCount))); + TimeUtil.randomDelay(1000); + continue; + } + LOGGER.info( + getLogByClass("Notify data server {} change data {} to sync,retry={}"), + connection.getRemoteIP(), request, tryCount); + syncServer.sendSync(syncServer.getChannel(connection.getRemoteAddress()), request, 1000); break; } catch (Exception e) { LOGGER.error(getLogByClass(String.format( - "Notify data server %s failed, NotifyDataSyncRequest:%s", targetDataIp, - request)), e); + "Notify data server %s failed, NotifyDataSyncRequest:%s,retry=%s", + targetDataIp, request, tryCount)), e); + TimeUtil.randomDelay(1000); } } } @@ -227,6 +246,8 @@ public void changeDataCheck() { removeCache(acceptor); // compare and remove } catch (InterruptedException e) { break; + } catch (Throwable e) { + LOGGER.error(e.getMessage(), e); } } @@ -263,11 +284,11 @@ public SyncData getSyncData(SyncDataRequest syncDataRequest) { } /** - * Getter method for property dataServerBootstrapConfig. + * Getter method for property dataServerConfig. * - * @return property value of dataServerBootstrapConfig + * @return property value of dataServerConfig */ public DataServerConfig getDataServerConfig() { - return dataServerBootstrapConfig; + return dataServerConfig; } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/Acceptor.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/Acceptor.java index 5650ea263..b758cf6fa 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/Acceptor.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/datasync/sync/Acceptor.java @@ -16,13 +16,6 @@ */ package com.alipay.sofa.registry.server.data.datasync.sync; -import com.alipay.sofa.registry.common.model.dataserver.Datum; -import com.alipay.sofa.registry.common.model.dataserver.SyncData; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.server.data.cache.DatumCache; -import com.alipay.sofa.registry.server.data.datasync.Operator; - import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; @@ -36,6 +29,14 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.dataserver.SyncData; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.datasync.Operator; +import com.alipay.sofa.registry.util.DatumVersionUtil; + /** * * @author shangyu.wh @@ -57,16 +58,19 @@ public class Acceptor { private final Lock read = readWriteLock.readLock(); private final Lock write = readWriteLock.writeLock(); + private final DatumCache datumCache; + /** * constructor * @param maxBufferSize * @param dataInfoId * @param dataCenter */ - public Acceptor(int maxBufferSize, String dataInfoId, String dataCenter) { + public Acceptor(int maxBufferSize, String dataInfoId, String dataCenter, DatumCache datumCache) { this.maxBufferSize = maxBufferSize; this.dataInfoId = dataInfoId; this.dataCenter = dataCenter; + this.datumCache = datumCache; } /** @@ -146,7 +150,7 @@ public SyncData process(Long currentVersion) { //first get all data if (operators.isEmpty()) { wholeDataTag = true; - retList.add(DatumCache.get(dataCenter, dataInfoId)); + retList.add(datumCache.get(dataCenter, dataInfoId)); LOGGER.info("Get all data!dataInfoID:{} dataCenter:{}.All data size{}:", dataInfoId, dataCenter, retList.size()); } else { @@ -162,7 +166,7 @@ public SyncData process(Long currentVersion) { "Append log queue is empty,Maybe all logs record expired or no operator append!So must get all data!dataInfoID:{} dataCenter:{}.queue size{}:", dataInfoId, dataCenter, logOperatorsOrder.size()); wholeDataTag = true; - retList.add(DatumCache.get(dataCenter, dataInfoId)); + retList.add(datumCache.get(dataCenter, dataInfoId)); syncData = new SyncData(dataInfoId, dataCenter, wholeDataTag, retList); } @@ -250,7 +254,8 @@ private boolean isFull() { private boolean isExpired(int durationSECS, long peekVersion) { durationSECS = (durationSECS > 0) ? durationSECS * 1000 : DEFAULT_DURATION_SECS * 1000; - boolean ret = System.currentTimeMillis() > peekVersion + durationSECS; + boolean ret = System.currentTimeMillis() > DatumVersionUtil.getRealTimestamp(peekVersion) + + durationSECS; if (LOGGER.isDebugEnabled()) { LOGGER.debug("now:" + System.currentTimeMillis() + " peek:" + peekVersion + " du:" + durationSECS + " result:" + ret); @@ -258,7 +263,7 @@ private boolean isExpired(int durationSECS, long peekVersion) { return ret; } - private void clearBefore() { + public void clearBefore() { write.lock(); try { diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/AfterWorkingProcess.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/AfterWorkingProcess.java new file mode 100644 index 000000000..f15aca14d --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/AfterWorkingProcess.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.event; + +/** + * + * @author shangyu.wh + * @version $Id: AfterWorkingProcessor.java, v 0.1 2019-05-21 11:25 shangyu.wh Exp $ + */ +public interface AfterWorkingProcess { + + void afterWorkingProcess(); + + int getOrder(); +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/DataServerChangeEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/DataServerChangeEvent.java index a41dfd113..9dfaa60f8 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/DataServerChangeEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/DataServerChangeEvent.java @@ -16,27 +16,37 @@ */ package com.alipay.sofa.registry.server.data.event; -import com.alipay.sofa.registry.common.model.metaserver.DataNode; -import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; - import java.util.HashMap; import java.util.Map; +import com.alipay.sofa.registry.common.model.metaserver.DataNode; +import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; + /** * * @author qian.lqlq * @version $Id: DataServerChangeEvent.java, v 0.1 2018-03-13 14:37 qian.lqlq Exp $ */ -public class DataServerChangeEvent { +public class DataServerChangeEvent implements Event { + + /** + * node type enum + */ + public enum FromType { + CONNECT_TASK, META_NOTIFY, REGISTER_META + } private DataServerChangeItem dataServerChangeItem; + private FromType fromType; + /** * constructor * @param dataServerChangeItem */ - public DataServerChangeEvent(DataServerChangeItem dataServerChangeItem) { + public DataServerChangeEvent(DataServerChangeItem dataServerChangeItem, FromType fromType) { this.dataServerChangeItem = dataServerChangeItem; + this.fromType = fromType; } /** @@ -45,7 +55,7 @@ public DataServerChangeEvent(DataServerChangeItem dataServerChangeItem) { * @param versionMap */ public DataServerChangeEvent(Map> serverMap, - Map versionMap) { + Map versionMap, FromType fromType) { if (serverMap == null) { serverMap = new HashMap<>(); } @@ -53,6 +63,8 @@ public DataServerChangeEvent(Map> serverMap, versionMap = new HashMap<>(); } this.dataServerChangeItem = new DataServerChangeItem(serverMap, versionMap); + + this.fromType = fromType; } /** @@ -63,4 +75,13 @@ public DataServerChangeEvent(Map> serverMap, public DataServerChangeItem getDataServerChangeItem() { return dataServerChangeItem; } + + /** + * Getter method for property fromType. + * + * @return property value of fromType + */ + public FromType getFromType() { + return fromType; + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/Event.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/Event.java new file mode 100644 index 000000000..1d79e6d49 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/Event.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.event; + +/** + * + * @author kezhu.wukz + * @version $Id: Event.java, v 0.1 2019-12-12 15:42 kezhu.wukz Exp $ + */ +public interface Event { +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/EventCenter.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/EventCenter.java index 3e5dd8daf..aa4da7d98 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/EventCenter.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/EventCenter.java @@ -16,10 +16,12 @@ */ package com.alipay.sofa.registry.server.data.event; -import com.alipay.sofa.registry.server.data.event.handler.AbstractEventHandler; +import java.util.Collection; +import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import com.alipay.sofa.registry.server.data.event.handler.AbstractEventHandler; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; /** * @@ -28,24 +30,32 @@ */ public class EventCenter { - private final Map MAP = new ConcurrentHashMap<>(); + private Multimap, AbstractEventHandler> MAP = ArrayListMultimap.create(); /** * eventHandler register * @param handler */ public void register(AbstractEventHandler handler) { - MAP.put(handler.interest(), handler); + List> interests = handler.interest(); + for (Class interest : interests) { + MAP.put(interest, handler); + } } /** * event handler handle process * @param event */ - public void post(Object event) { + public void post(Event event) { Class clazz = event.getClass(); if (MAP.containsKey(clazz)) { - MAP.get(clazz).handle(event); + Collection handlers = MAP.get(clazz); + if (handlers != null) { + for (AbstractEventHandler handler : handlers) { + handler.handle(event); + } + } } else { throw new RuntimeException("no suitable handler was found:" + clazz); } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/LocalDataServerChangeEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/LocalDataServerChangeEvent.java index 100af3482..8a89432db 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/LocalDataServerChangeEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/LocalDataServerChangeEvent.java @@ -16,17 +16,17 @@ */ package com.alipay.sofa.registry.server.data.event; -import com.alipay.sofa.registry.common.model.metaserver.DataNode; - import java.util.Map; import java.util.Set; +import com.alipay.sofa.registry.common.model.metaserver.DataNode; + /** * * @author qian.lqlq * @version $Id: LocalDataServerChangeEvent.java, v 0.1 2018-05-07 20:13 qian.lqlq Exp $ */ -public class LocalDataServerChangeEvent { +public class LocalDataServerChangeEvent implements Event { private Map localDataServerMap; @@ -87,4 +87,18 @@ public Set getNewJoined() { public long getLocalDataCenterversion() { return localDataCenterversion; } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LocalDataServerChangeEvent{"); + sb.append("localDataServerMap=").append(localDataServerMap); + sb.append(", localDataCenterversion=").append(localDataCenterversion); + sb.append(", newJoined=").append(newJoined); + sb.append(", version=").append(version); + sb.append('}'); + return sb.toString(); + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/MetaServerChangeEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/MetaServerChangeEvent.java index d726eb664..6057f0f07 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/MetaServerChangeEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/MetaServerChangeEvent.java @@ -24,7 +24,7 @@ * @author qian.lqlq * @version $Id: MetaServerChangeEvent.java, v 0.1 2018-03-13 15:31 qian.lqlq Exp $ */ -public class MetaServerChangeEvent { +public class MetaServerChangeEvent implements Event { private Map> ipMap; diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/RemoteDataServerChangeEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/RemoteDataServerChangeEvent.java new file mode 100644 index 000000000..dd57ed815 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/RemoteDataServerChangeEvent.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.event; + +import java.util.Map; + +import com.alipay.sofa.registry.common.model.metaserver.DataNode; + +/** + * + * @author kezhu.wukz + * @version $Id: RemoteDataServerChangeEvent.java, v 0.1 2019-12-17 20:13 kezhu.wukz Exp $ + */ +public class RemoteDataServerChangeEvent implements Event { + + private final String dataCenter; + private Map remoteDataServerMap; + + private long remoteDataCenterVersion; + + private long version; + + /** + * constructor + * @param remoteDataServerMap + * @param version + * @param remoteDataCenterVersion + */ + public RemoteDataServerChangeEvent(String dataCenter, + Map remoteDataServerMap, long version, + long remoteDataCenterVersion) { + this.dataCenter = dataCenter; + this.remoteDataServerMap = remoteDataServerMap; + this.version = version; + this.remoteDataCenterVersion = remoteDataCenterVersion; + } + + /** + * Getter method for property dataCenter. + * + * @return property value of dataCenter + */ + public String getDataCenter() { + return dataCenter; + } + + /** + * Getter method for property remoteDataServerMap. + * + * @return property value of remoteDataServerMap + */ + public Map getRemoteDataServerMap() { + return remoteDataServerMap; + } + + /** + * Getter method for property version. + * + * @return property value of version + */ + public long getVersion() { + return version; + } + + /** + * Getter method for property remoteDataCenterVersion. + * + * @return property value of remoteDataCenterVersion + */ + public long getRemoteDataCenterVersion() { + return remoteDataCenterVersion; + } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RemoteDataServerChangeEvent{"); + sb.append("dataCenter='").append(dataCenter).append('\''); + sb.append(", remoteDataServerMap=").append(remoteDataServerMap); + sb.append(", remoteDataCenterVersion=").append(remoteDataCenterVersion); + sb.append(", version=").append(version); + sb.append('}'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskEvent.java index f49f3c01a..23591b6b6 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskEvent.java @@ -16,24 +16,28 @@ */ package com.alipay.sofa.registry.server.data.event; +import java.util.Set; + /** * * @author qian.lqlq * @version $Id: StartTaskEvent.java, v 0.1 2018-03-13 15:13 qian.lqlq Exp $ */ -public class StartTaskEvent { +public class StartTaskEvent implements Event { - private static final StartTaskEvent INSTANCE = new StartTaskEvent(); + private final Set suitableTypes; - private StartTaskEvent() { + public StartTaskEvent(Set suitableTypes) { + this.suitableTypes = suitableTypes; } /** - * get instance of StartTaskEvent - * @return + * Getter method for property suitableTypes. + * + * @return property value of suitableTypes */ - public static StartTaskEvent getInstance() { - return INSTANCE; + public Set getSuitableTypes() { + return suitableTypes; } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskTypeEnum.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskTypeEnum.java new file mode 100644 index 000000000..a43e026f2 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/StartTaskTypeEnum.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.event; + +/** + * + * @author shangyu.wh + * @version $Id: StartTaskTypeEnum.java, v 0.1 2019-04-22 14:35 shangyu.wh Exp $ + */ +public enum StartTaskTypeEnum { + + /** + * ConnectionRefreshMetaTask + */ + CONNECT_META, + + /** + * ConnectionRefreshDataTask + */ + CONNECT_DATA, + + /** + * RenewNodeTask + */ + RENEW, + + /** + * VersionCompareTask + */ + VERSION_COMPARE, + + LOG_METRICS +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AbstractEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AbstractEventHandler.java index e33401b61..6b79d868e 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AbstractEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AbstractEventHandler.java @@ -16,18 +16,21 @@ */ package com.alipay.sofa.registry.server.data.event.handler; +import java.util.List; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.data.event.EventCenter; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; /** * * @author qian.lqlq * @version $Id: AbstractEventHandler.java, v 0.1 2018-03-13 15:34 qian.lqlq Exp $ */ -public abstract class AbstractEventHandler implements InitializingBean { +public abstract class AbstractEventHandler implements InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractEventHandler.class); @@ -43,7 +46,7 @@ public void afterPropertiesSet() throws Exception { * event handle func * @param event */ - public void handle(T event) { + public void handle(Event event) { try { doHandle(event); } catch (Exception e) { @@ -51,7 +54,7 @@ public void handle(T event) { } } - public abstract Class interest(); + public abstract List> interest(); - public abstract void doHandle(T event); + public abstract void doHandle(Event event); } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AfterWorkingProcessHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AfterWorkingProcessHandler.java new file mode 100644 index 000000000..3e7f3cfba --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/AfterWorkingProcessHandler.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.event.handler; + +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; + +import javax.annotation.Resource; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +/** + * + * @author shangyu.wh + * @version $Id: AfterWorkingProcessHandler.java, v 0.1 2019-05-21 11:46 shangyu.wh Exp $ + */ +public class AfterWorkingProcessHandler implements AfterWorkingProcess { + + @Resource(name = "afterWorkProcessors") + private List afterWorkingProcessors; + + @Override + public void afterWorkingProcess() { + + if(afterWorkingProcessors != null){ + List list = afterWorkingProcessors.stream().sorted(Comparator.comparing(AfterWorkingProcess::getOrder)).collect(Collectors.toList()); + + list.forEach(AfterWorkingProcess::afterWorkingProcess); + } + } + + @Override + public int getOrder() { + return 0; + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/DataServerChangeEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/DataServerChangeEventHandler.java index 931877b03..a6fee6010 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/DataServerChangeEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/DataServerChangeEventHandler.java @@ -16,6 +16,18 @@ */ package com.alipay.sofa.registry.server.data.event.handler; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.common.model.store.URL; @@ -28,19 +40,12 @@ import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent; import com.alipay.sofa.registry.server.data.event.EventCenter; import com.alipay.sofa.registry.server.data.event.LocalDataServerChangeEvent; +import com.alipay.sofa.registry.server.data.event.RemoteDataServerChangeEvent; import com.alipay.sofa.registry.server.data.node.DataServerNode; import com.alipay.sofa.registry.server.data.remoting.DataNodeExchanger; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; import com.alipay.sofa.registry.server.data.util.TimeUtil; -import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.CollectionUtils; - -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.Lists; /** * @@ -55,7 +60,7 @@ public class DataServerChangeEventHandler extends AbstractEventHandler> interest() { + return Lists.newArrayList(DataServerChangeEvent.class); } @Override public void doHandle(DataServerChangeEvent event) { synchronized (this) { //register self first,execute once - DataServerNodeFactory.initConsistent(dataServerBootstrapConfig); + DataServerNodeFactory.initConsistent(dataServerConfig); DataServerChangeItem dataServerChangeItem = event.getDataServerChangeItem(); Set localDataServers = dataServerCache.getDataServers( - dataServerBootstrapConfig.getLocalDataCenter()).keySet(); + dataServerConfig.getLocalDataCenter()).keySet(); //get changed dataservers - Map> changedMap = dataServerCache - .compareAndSet(dataServerChangeItem); - if(!changedMap.isEmpty()) { + Map> changedMap = dataServerCache.compareAndSet( + dataServerChangeItem, event.getFromType()); + if (!changedMap.isEmpty()) { for (Entry> changeEntry : changedMap.entrySet()) { String dataCenter = changeEntry.getKey(); Set ips = changeEntry.getValue(); + Long newVersion = dataServerCache.getDataCenterNewVersion(dataCenter); if (!CollectionUtils.isEmpty(ips)) { for (String ip : ips) { if (!StringUtils.equals(ip, DataServerConfig.IP)) { DataServerNode dataServerNode = DataServerNodeFactory - .getDataServerNode(dataCenter, ip); - if (dataServerNode == null || dataServerNode.getConnection() == null - || !dataServerNode.getConnection().isFine()) { + .getDataServerNode(dataCenter, ip); + if (dataServerNode == null + || dataServerNode.getConnection() == null + || !dataServerNode.getConnection().isFine()) { connectDataServer(dataCenter, ip); } } @@ -102,58 +109,71 @@ public void doHandle(DataServerChangeEvent event) { Set ipSet = DataServerNodeFactory.getIps(dataCenter); for (String ip : ipSet) { if (!ips.contains(ip)) { - DataServerNodeFactory.remove(dataCenter, ip, dataServerBootstrapConfig); - LOGGER.info( - "[DataServerChangeEventHandler] remove connection, datacenter:{}, ip:{}", - dataCenter, ip); + DataServerNodeFactory.remove(dataCenter, ip, dataServerConfig); + LOGGER + .info( + "[DataServerChangeEventHandler] remove connection, datacenter:{}, ip:{},from:{}", + dataCenter, ip, event.getFromType()); } } - Long newVersion = dataServerCache.getDataCenterNewVersion(dataCenter); - Map newDataNodes = dataServerCache.getNewDataServerMap(dataCenter); - //if the datacenter is self, post LocalDataServerChangeEvent - if (dataServerBootstrapConfig.getLocalDataCenter().equals(dataCenter)) { + Map newDataNodes = dataServerCache + .getNewDataServerMap(dataCenter); + + //avoid input map reference operation DataServerNodeFactory MAP + Map map = new ConcurrentHashMap<>(newDataNodes); + + //if the dataCenter is self, post LocalDataServerChangeEvent + if (dataServerConfig.isLocalDataCenter(dataCenter)) { Set newjoined = new HashSet<>(ips); newjoined.removeAll(localDataServers); - //avoid input map reference operation DataServerNodeFactory MAP - Map map = new ConcurrentHashMap<>(newDataNodes); - - LOGGER.info("Node list change fire LocalDataServerChangeEvent,current node list={},version={}", - map.keySet(), newVersion); + LOGGER + .info( + "Node list change fire LocalDataServerChangeEvent,current node list={},version={},from:{}", + map.keySet(), newVersion, event.getFromType()); eventCenter.post(new LocalDataServerChangeEvent(map, newjoined, - dataServerChangeItem.getVersionMap() - .get(dataServerBootstrapConfig.getLocalDataCenter()), - newVersion)); + dataServerChangeItem.getVersionMap().get(dataCenter), newVersion)); } else { dataServerCache.updateItem(newDataNodes, newVersion, dataCenter); + eventCenter.post(new RemoteDataServerChangeEvent(dataCenter, map, + dataServerChangeItem.getVersionMap().get(dataCenter), newVersion)); } } else { - //if the datacenter which has no dataservers is not self, remove it - if (!dataServerBootstrapConfig.getLocalDataCenter().equals(dataCenter)) { + //if the dataCenter which has no dataServers is not self, remove it + if (!dataServerConfig.isLocalDataCenter(dataCenter)) { removeDataCenter(dataCenter); + eventCenter.post(new RemoteDataServerChangeEvent(dataCenter, + Collections.EMPTY_MAP, dataServerChangeItem.getVersionMap().get( + dataCenter), newVersion)); } - Long newVersion = dataServerCache.getDataCenterNewVersion(dataCenter); - Map newDataNodes = dataServerCache.getNewDataServerMap(dataCenter); + Map newDataNodes = dataServerCache + .getNewDataServerMap(dataCenter); dataServerCache.updateItem(newDataNodes, newVersion, dataCenter); } } } else { - //refresh for keep connect - Set allDataCenter = new HashSet<>(DataServerNodeFactory.getAllDataCenters()); - for (String dataCenter:allDataCenter) { - Map dataNodes = DataServerNodeFactory.getDataServerNodes(dataCenter); - if(dataNodes != null && !dataNodes.isEmpty()){ - - dataNodes.forEach((ip,dataServerNode)->{ - if (!StringUtils.equals(ip, DataServerConfig.IP)) { - Connection connection = dataServerNode.getConnection(); - if (connection != null && !connection.isFine()) { - LOGGER.warn("[DataServerChangeEventHandler] dataServer connections is not fine,try to reconnect it,old connection={},dataCenter={}", - connection.getRemoteAddress(), dataCenter); - connectDataServer(dataCenter, ip); + //refresh for keep connect other dataServers + Set allDataCenter = new HashSet<>(dataServerCache.getAllDataCenters()); + for (String dataCenter : allDataCenter) { + Map dataNodes = dataServerCache + .getNewDataServerMap(dataCenter); + if (dataNodes != null) { + for (DataNode dataNode : dataNodes.values()) { + if (!StringUtils.equals(dataNode.getIp(), DataServerConfig.IP)) { + DataServerNode dataServerNode = DataServerNodeFactory + .getDataServerNode(dataCenter, dataNode.getIp()); + Connection connection = dataServerNode != null ? dataServerNode + .getConnection() : null; + if (connection == null || !connection.isFine()) { + LOGGER + .warn( + "[DataServerChangeEventHandler] dataServer connections is not fine, try to reconnect it, old connection={}, dataNode={}, dataCenter={}, from:{}", + connection, dataNode.getIp(), dataCenter, + event.getFromType()); + connectDataServer(dataCenter, dataNode.getIp()); } } - }); + } } } } @@ -170,8 +190,8 @@ private void connectDataServer(String dataCenter, String ip) { Connection conn = null; for (int tryCount = 0; tryCount < TRY_COUNT; tryCount++) { try { - conn = ((BoltChannel) dataNodeExchanger.connect(new URL(ip, - dataServerBootstrapConfig.getSyncDataPort()))).getConnection(); + conn = ((BoltChannel) dataNodeExchanger.connect(new URL(ip, dataServerConfig + .getSyncDataPort()))).getConnection(); break; } catch (Exception e) { LOGGER.error("[DataServerChangeEventHandler] connect dataServer {} in {} error", @@ -181,21 +201,20 @@ private void connectDataServer(String dataCenter, String ip) { } if (conn == null || !conn.isFine()) { LOGGER.error( - "[DataServerChangeEventHandler] connect dataserver {} in {} failed five times", ip, + "[DataServerChangeEventHandler] connect dataServer {} in {} failed five times", ip, dataCenter); throw new RuntimeException( String .format( - "[DataServerChangeEventHandler] connect dataserver %s in %s failed five times,dataServer will not work,please check connect!", + "[DataServerChangeEventHandler] connect dataServer %s in %s failed five times,dataServer will not work,please check connect!", ip, dataCenter)); } //maybe get dataNode from metaServer,current has not start! register dataNode info to factory,wait for connect task next execute - DataServerNodeFactory.register(new DataServerNode(ip, dataCenter, conn), - dataServerBootstrapConfig); + DataServerNodeFactory.register(new DataServerNode(ip, dataCenter, conn), dataServerConfig); } /** - * remove datacenter, and close connections of dataservers in this datacenter + * remove dataCenter, and close connections of dataServers in this dataCenter * * @param dataCenter */ @@ -203,7 +222,6 @@ private void removeDataCenter(String dataCenter) { DataServerNodeFactory.getDataServerNodes(dataCenter).values().stream().map(DataServerNode::getConnection) .filter(connection -> connection != null && connection.isFine()).forEach(Connection::close); DataServerNodeFactory.remove(dataCenter); - LOGGER.info( - "[DataServerChangeEventHandler] remove connections of datacenter : {}", dataCenter); + LOGGER.info("[DataServerChangeEventHandler] remove connections of dataCenter : {}", dataCenter); } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/LocalDataServerChangeEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/LocalDataServerChangeEventHandler.java index 774b7572f..67a01d65b 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/LocalDataServerChangeEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/LocalDataServerChangeEventHandler.java @@ -16,6 +16,19 @@ */ package com.alipay.sofa.registry.server.data.event.handler; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.NotifyFetchDatumRequest; @@ -25,33 +38,23 @@ import com.alipay.sofa.registry.consistency.hash.ConsistentHash; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.bolt.BoltChannel; import com.alipay.sofa.registry.remoting.exchange.message.Request; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.cache.BackupTriad; import com.alipay.sofa.registry.server.data.cache.DataServerCache; import com.alipay.sofa.registry.server.data.cache.DatumCache; -import com.alipay.sofa.registry.server.data.correction.LocalDataServerCleanHandler; import com.alipay.sofa.registry.server.data.event.LocalDataServerChangeEvent; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.node.DataNodeStatus; import com.alipay.sofa.registry.server.data.node.DataServerNode; import com.alipay.sofa.registry.server.data.remoting.DataNodeExchanger; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; +import com.alipay.sofa.registry.server.data.renew.LocalDataServerCleanHandler; import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; import com.alipay.sofa.registry.server.data.util.TimeUtil; import com.google.common.collect.Lists; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.atomic.AtomicBoolean; /** * @@ -66,7 +69,7 @@ public class LocalDataServerChangeEventHandler extends .getLogger(LocalDataServerChangeEventHandler.class); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private LocalDataServerCleanHandler localDataServerCleanHandler; @@ -80,19 +83,31 @@ public class LocalDataServerChangeEventHandler extends @Autowired private DataNodeStatus dataNodeStatus; + @Autowired + private DatumCache datumCache; + + @Autowired + private DatumLeaseManager datumLeaseManager; + private BlockingQueue events = new LinkedBlockingDeque<>(); private AtomicBoolean isChanged = new AtomicBoolean(false); + private static final int TRY_COUNT = 5; + @Override - public Class interest() { - return LocalDataServerChangeEvent.class; + public List> interest() { + return Lists.newArrayList(LocalDataServerChangeEvent.class); } @Override public void doHandle(LocalDataServerChangeEvent localDataServerChangeEvent) { isChanged.set(true); + + // Better change to Listener pattern localDataServerCleanHandler.reset(); + datumLeaseManager.reset(); + events.offer(localDataServerChangeEvent); } @@ -119,11 +134,11 @@ public void run() { try { LocalDataServerChangeEvent event = events.take(); //if the new joined servers contains self, set status as INITIAL - Set newJoined = event.getNewJoined(); - if (newJoined.contains(DataServerConfig.IP) - && dataNodeStatus.getStatus() != LocalServerStatusEnum.INITIAL) { - dataNodeStatus.setStatus(LocalServerStatusEnum.INITIAL); - } + //Set newJoined = event.getNewJoined(); + //if (newJoined.contains(DataServerConfig.IP) + // && dataNodeStatus.getStatus() != LocalServerStatusEnum.INITIAL) { + // dataNodeStatus.setStatus(LocalServerStatusEnum.INITIAL); + //} //if size of events is greater than 0, not handle and continue, only handle the last one in the queue if (events.size() > 0) { continue; @@ -143,7 +158,7 @@ public void run() { dataServerCache.updateItem(event.getLocalDataServerMap(), event.getLocalDataCenterversion(), - dataServerBootstrapConfig.getLocalDataCenter()); + dataServerConfig.getLocalDataCenter()); } } catch (Throwable t) { LOGGER.error("sync local data error", t); @@ -162,7 +177,7 @@ private void notifyToFetch(LocalDataServerChangeEvent event, long changeVersion) Map dataServerMapIn = event.getLocalDataServerMap(); List dataServerNodeList = Lists.newArrayList(dataServerMapIn.values()); ConsistentHash consistentHash = new ConsistentHash<>( - dataServerBootstrapConfig.getNumberOfReplicas(), dataServerNodeList); + dataServerConfig.getNumberOfReplicas(), dataServerNodeList); Map dataServerMap = new ConcurrentHashMap<>(dataServerMapIn); Map>> toBeSyncMap = getToBeSyncMap(consistentHash); @@ -182,7 +197,7 @@ private void notifyToFetch(LocalDataServerChangeEvent event, long changeVersion) for (Entry dataTriadEntry : dataTriadMap .entrySet()) { String dataInfoId = dataTriadEntry.getKey(); - Datum datum = DatumCache.get(dataCenter, dataInfoId); + Datum datum = datumCache.get(dataCenter, dataInfoId); if (datum != null) { versionMap.put(dataInfoId, datum.getVersion()); } @@ -212,7 +227,7 @@ private void notifyToFetch(LocalDataServerChangeEvent event, long changeVersion) if (!isChanged.get()) { //update server list dataServerCache.updateItem(dataServerMapIn, event.getLocalDataCenterversion(), - dataServerBootstrapConfig.getLocalDataCenter()); + dataServerConfig.getLocalDataCenter()); } } } @@ -228,8 +243,11 @@ private void notifyToFetch(LocalDataServerChangeEvent event, long changeVersion) Map>> toBeSyncMap = new HashMap<>(); Map> triadCache = new HashMap<>(); + ConsistentHash consistentHashOld = dataServerCache + .calculateOldConsistentHash(dataServerConfig.getLocalDataCenter()); + //compute new triad for every datum in cache - Map> allMap = DatumCache.getAll(); + Map> allMap = datumCache.getAll(); for (Entry> dataCenterEntry : allMap.entrySet()) { String dataCenter = dataCenterEntry.getKey(); Map datumMap = dataCenterEntry.getValue(); @@ -244,11 +262,12 @@ private void notifyToFetch(LocalDataServerChangeEvent event, long changeVersion) backupNodes = triadCache.get(dataInfoId); } else { backupNodes = consistentHash.getNUniqueNodesFor(dataInfoId, - dataServerBootstrapConfig.getStoreNodes()); + dataServerConfig.getStoreNodes()); triadCache.put(dataInfoId, backupNodes); } - BackupTriad backupTriad = dataServerCache.calculateOldBackupTriad(dataInfoId, - dataServerBootstrapConfig.getLocalDataCenter(), dataServerBootstrapConfig); + BackupTriad backupTriad = new BackupTriad(dataInfoId, + consistentHashOld.getNUniqueNodesFor(dataInfoId, + dataServerConfig.getStoreNodes())); if (backupTriad != null) { List newJoinedNodes = backupTriad.getNewJoined(backupNodes, dataServerCache.getNotWorking()); @@ -288,7 +307,7 @@ private boolean doNotify(String targetIp, Map> notifyV long version) { while (!isChanged.get()) { DataServerNode targetNode = DataServerNodeFactory.getDataServerNode( - dataServerBootstrapConfig.getLocalDataCenter(), targetIp); + dataServerConfig.getLocalDataCenter(), targetIp); if (targetNode == null || targetNode.getConnection() == null) { LOGGER.info( "notify version change to sync has not connect,targetNode={}, map={}", @@ -335,17 +354,37 @@ public URL getRequestUrl() { */ private void notifyOnline(long changeVersion) { Map dataServerNodeMap = DataServerNodeFactory - .getDataServerNodes(dataServerBootstrapConfig.getLocalDataCenter()); + .getDataServerNodes(dataServerConfig.getLocalDataCenter()); for (Entry serverEntry : dataServerNodeMap.entrySet()) { while (true) { String ip = serverEntry.getKey(); - DataServerNode dataServerNode = serverEntry.getValue(); + DataServerNode dataServerNode = DataServerNodeFactory.getDataServerNode( + dataServerConfig.getLocalDataCenter(), ip); + if (dataServerNode == null) { + LOGGER + .warn( + "notify Online dataserver {} has not existed in DataServerNodeFactory!version={}", + ip, changeVersion); break; } try { - if (dataServerNode.getConnection() == null - || !dataServerNode.getConnection().isFine()) { + final Connection connection = dataServerNode.getConnection(); + if (connection == null || !connection.isFine()) { + if (connection == null) { + LOGGER + .warn( + "notify Online dataserver connect not existed,ip={},version={}", + ip, changeVersion); + } else { + LOGGER + .warn( + "notify Online dataserver connect not fine!remote={},local={},version={}", + connection.getRemoteAddress(), + connection.getLocalAddress(), changeVersion); + } + //connect now and registry connect + connectDataServer(dataServerConfig.getLocalDataCenter(), ip); //maybe get dataNode from metaServer,current has not connected!wait for connect task execute TimeUtil.randomDelay(1000); continue; @@ -361,8 +400,8 @@ public Object getRequestBody() { @Override public URL getRequestUrl() { - return new URL(dataServerNode.getConnection().getRemoteIP(), - dataServerNode.getConnection().getRemotePort()); + return new URL(connection.getRemoteIP(), connection + .getRemotePort()); } }).getResult(); if (response.isSuccess()) { @@ -381,5 +420,44 @@ public URL getRequestUrl() { } } + /** + * connect specific dataserver + * + * @param dataCenter + * @param ip + */ + private void connectDataServer(String dataCenter, String ip) { + Connection conn = null; + for (int tryCount = 0; tryCount < TRY_COUNT; tryCount++) { + try { + conn = ((BoltChannel) dataNodeExchanger.connect(new URL(ip, dataServerConfig + .getSyncDataPort()))).getConnection(); + break; + } catch (Exception e) { + LOGGER.error( + "[LocalDataServerChangeEventHandler] connect dataServer {} in {} error", + ip, dataCenter, e); + TimeUtil.randomDelay(3000); + } + } + if (conn == null || !conn.isFine()) { + LOGGER + .error( + "[LocalDataServerChangeEventHandler] connect dataserver {} in {} failed five times", + ip, dataCenter); + throw new RuntimeException( + String + .format( + "[LocalDataServerChangeEventHandler] connect dataserver %s in %s failed five times,dataServer will not work,please check connect!", + ip, dataCenter)); + } + LOGGER + .info( + "[LocalDataServerChangeEventHandler] connect dataserver in {} success,remote={},local={}", + dataCenter, conn.getRemoteAddress(), conn.getLocalAddress()); + //maybe get dataNode from metaServer,current has not start! register dataNode info to factory,wait for connect task next execute + DataServerNodeFactory.register(new DataServerNode(ip, dataCenter, conn), + dataServerConfig); + } } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/MetaServerChangeEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/MetaServerChangeEventHandler.java index 3518503cb..b3d98de26 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/MetaServerChangeEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/MetaServerChangeEventHandler.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.data.event.handler; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; + import com.alipay.remoting.Connection; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.registry.common.model.metaserver.DataNode; @@ -30,16 +39,13 @@ import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent; import com.alipay.sofa.registry.server.data.event.EventCenter; import com.alipay.sofa.registry.server.data.event.MetaServerChangeEvent; +import com.alipay.sofa.registry.server.data.event.StartTaskEvent; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import com.alipay.sofa.registry.server.data.remoting.MetaNodeExchanger; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import com.alipay.sofa.registry.server.data.remoting.metaserver.MetaServerConnectionFactory; import com.alipay.sofa.registry.server.data.util.TimeUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.CollectionUtils; - -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; +import com.google.common.collect.Lists; /** * @@ -53,7 +59,7 @@ public class MetaServerChangeEventHandler extends AbstractEventHandler> interest() { + return Lists.newArrayList(MetaServerChangeEvent.class); } @Override @@ -116,7 +122,7 @@ private void registerMetaServer(String dataCenter, String ip) { for (int tryCount = 0; tryCount < TRY_COUNT; tryCount++) { try { - Channel channel = metaNodeExchanger.connect(new URL(ip, dataServerBootstrapConfig + Channel channel = metaNodeExchanger.connect(new URL(ip, dataServerConfig .getMetaServerPort())); //connect all meta server if (channel != null && channel.isConnected()) { @@ -130,13 +136,13 @@ private void registerMetaServer(String dataCenter, String ip) { obj = metaNodeExchanger.request(new Request() { @Override public Object getRequestBody() { - return new DataNode(new URL(DataServerConfig.IP), - dataServerBootstrapConfig.getLocalDataCenter()); + return new DataNode(new URL(DataServerConfig.IP), dataServerConfig + .getLocalDataCenter()); } @Override public URL getRequestUrl() { - return new URL(ip, dataServerBootstrapConfig.getMetaServerPort()); + return new URL(ip, dataServerConfig.getMetaServerPort()); } }).getResult(); } catch (Exception e) { @@ -144,13 +150,22 @@ public URL getRequestUrl() { LOGGER .error( - "[MetaServerChangeEventHandler] register data node send error!retry once leader :{} error", - newLeader.getIp(), e); + String + .format( + "[MetaServerChangeEventHandler] register data node send error!retry once leader :%s error", + newLeader.getIp()), e); } if (obj instanceof NodeChangeResult) { NodeChangeResult result = (NodeChangeResult) obj; Map versionMap = result.getDataCenterListVersions(); - eventCenter.post(new DataServerChangeEvent(result.getNodes(), versionMap)); + + //send renew after first register dataNode + Set set = new HashSet<>(); + set.add(StartTaskTypeEnum.RENEW); + eventCenter.post(new StartTaskEvent(set)); + + eventCenter.post(new DataServerChangeEvent(result.getNodes(), versionMap, + DataServerChangeEvent.FromType.REGISTER_META)); break; } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/StartTaskEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/StartTaskEventHandler.java index c6adfffe1..886aaf59f 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/StartTaskEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/event/handler/StartTaskEventHandler.java @@ -16,16 +16,17 @@ */ package com.alipay.sofa.registry.server.data.event.handler; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; + +import javax.annotation.Resource; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.server.data.cache.CacheDigestTask; import com.alipay.sofa.registry.server.data.event.StartTaskEvent; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.remoting.dataserver.task.AbstractTask; - -import javax.annotation.Resource; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; +import com.google.common.collect.Lists; /** * @@ -34,32 +35,39 @@ */ public class StartTaskEventHandler extends AbstractEventHandler { - private static final Logger LOGGER = LoggerFactory - .getLogger(StartTaskEventHandler.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(StartTaskEventHandler.class); + + private static final Logger LOGGER_START = LoggerFactory.getLogger("DATA-START-LOGS"); @Resource(name = "tasks") private List tasks; - private ScheduledExecutorService executor = null; + private ScheduledExecutorService executor = null; @Override - public Class interest() { - return StartTaskEvent.class; + public List> interest() { + return Lists.newArrayList(StartTaskEvent.class); } @Override public void doHandle(StartTaskEvent event) { if (executor == null || executor.isShutdown()) { - executor = ExecutorFactory.newScheduledThreadPool(tasks.size(), this.getClass() - .getSimpleName()); - for (AbstractTask task : tasks) { - LOGGER.info("[StartTaskEventHandler] start task:{}", task.getName()); + getExecutor(); + } + + for (AbstractTask task : tasks) { + if (event.getSuitableTypes().contains(task.getStartTaskTypeEnum())) { executor.scheduleWithFixedDelay(task, task.getInitialDelay(), task.getDelay(), task.getTimeUnit()); - LOGGER.info("[StartTaskEventHandler] start task:{} success", task.getName()); + LOGGER_START.info("[StartTaskEventHandler] start task:{} success", task.getName()); } - new CacheDigestTask().start(); } } + private void getExecutor() { + executor = ExecutorFactory.newScheduledThreadPool(tasks.size(), this.getClass() + .getSimpleName()); + } + } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/executor/ExecutorFactory.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/executor/ExecutorFactory.java index 233e6013c..c8ff3b4d9 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/executor/ExecutorFactory.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/executor/ExecutorFactory.java @@ -16,10 +16,6 @@ */ package com.alipay.sofa.registry.server.data.executor; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.util.NamedThreadFactory; - import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; @@ -29,6 +25,11 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.util.DataMetricsThreadPoolExecutor; +import com.alipay.sofa.registry.util.NamedThreadFactory; + /** * the factory to create executor * @@ -38,12 +39,13 @@ public class ExecutorFactory { public static final ThreadPoolExecutor EXECUTOR; + public static final ThreadPoolExecutor NOTIFY_SESSION_CALLBACK_EXECUTOR; private static final Logger LOGGER = LoggerFactory.getLogger(ExecutorFactory.class); static { BlockingQueue workQueue = new ArrayBlockingQueue<>(10); - EXECUTOR = new ThreadPoolExecutor(20, 300, 1, TimeUnit.HOURS, workQueue, - new NamedThreadFactory("CommonExecutor")) { + EXECUTOR = new DataMetricsThreadPoolExecutor("CommonExecutor", 20, 300, 1, TimeUnit.HOURS, + workQueue, new NamedThreadFactory("CommonExecutor")) { /** * @see ThreadPoolExecutor#afterExecute(Runnable, Throwable) @@ -57,6 +59,12 @@ protected void afterExecute(Runnable r, Throwable t) { } } }; + + NOTIFY_SESSION_CALLBACK_EXECUTOR = new DataMetricsThreadPoolExecutor( + "NotifySessionCallbackExecutor", 10, 20, 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(100000), new NamedThreadFactory( + "NotifySessionCallbackExecutor", true)); + } /** diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/DataNodeExchanger.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/DataNodeExchanger.java index 23b9865c3..a87bd7221 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/DataNodeExchanger.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/DataNodeExchanger.java @@ -16,6 +16,12 @@ */ package com.alipay.sofa.registry.server.data.remoting; +import java.util.Collection; + +import javax.annotation.Resource; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -28,10 +34,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Resource; -import java.util.Collection; /** * @author xuanbei @@ -45,26 +47,25 @@ public class DataNodeExchanger implements NodeExchanger { private Exchange boltExchange; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Resource(name = "dataClientHandlers") private Collection dataClientHandlers; @Override public Response request(Request request) { - Channel channel = this.connect(request.getRequestUrl()); Client client = boltExchange.getClient(Exchange.DATA_SERVER_TYPE); LOGGER.info("DataNode Exchanger request={},url={},callbackHandler={}", request.getRequestBody(), request.getRequestUrl(), request.getCallBackHandler()); if (null != request.getCallBackHandler()) { - client.sendCallback(channel, request.getRequestBody(), + client.sendCallback(request.getRequestUrl(), request.getRequestBody(), request.getCallBackHandler(), - dataServerBootstrapConfig.getRpcTimeout()); + request.getTimeout() != null ? request.getTimeout() : dataServerConfig.getRpcTimeout()); return () -> Response.ResultStatus.SUCCESSFUL; } else { - final Object result = client.sendSync(channel, request.getRequestBody(), - dataServerBootstrapConfig.getRpcTimeout()); + final Object result = client.sendSync(request.getRequestUrl(), request.getRequestBody(), + dataServerConfig.getRpcTimeout()); return () -> result; } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/MetaNodeExchanger.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/MetaNodeExchanger.java index 607beec97..b60f1c609 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/MetaNodeExchanger.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/MetaNodeExchanger.java @@ -16,6 +16,12 @@ */ package com.alipay.sofa.registry.server.data.remoting; +import java.util.Collection; + +import javax.annotation.Resource; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -29,10 +35,6 @@ import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Resource; -import java.util.Collection; /** * @author xuanbei @@ -49,34 +51,29 @@ public class MetaNodeExchanger implements NodeExchanger { private IMetaServerService metaServerService; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Resource(name = "metaClientHandlers") private Collection metaClientHandlers; @Override public Response request(Request request) { - Channel channel = connect(request.getRequestUrl()); Client client = boltExchange.getClient(Exchange.META_SERVER_TYPE); LOGGER.info("MetaNode Exchanger request={},url={},callbackHandler={}", request.getRequestBody(), request.getRequestUrl(), request.getCallBackHandler()); try { - final Object result = client.sendSync(channel, request.getRequestBody(), - dataServerBootstrapConfig.getRpcTimeout()); + final Object result = client.sendSync(request.getRequestUrl(), request.getRequestBody(), + dataServerConfig.getRpcTimeout()); return () -> result; } catch (Exception e) { //retry URL url = new URL(metaServerService.refreshLeader().getIp(), - dataServerBootstrapConfig.getMetaServerPort()); - channel = client.getChannel(url); - if (channel == null) { - channel = client.connect(url); - } + dataServerConfig.getMetaServerPort()); LOGGER.warn("MetaNode Exchanger request send error!It will be retry once!Request url:{}", url); - final Object result = client.sendSync(channel, request.getRequestBody(), - dataServerBootstrapConfig.getRpcTimeout()); + final Object result = client.sendSync(url, request.getRequestBody(), + request.getTimeout() != null ? request.getTimeout() : dataServerConfig.getRpcTimeout()); return () -> result; } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerConnectionFactory.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerConnectionFactory.java index 6cb2d8f9b..c7f62a93f 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerConnectionFactory.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerConnectionFactory.java @@ -31,7 +31,7 @@ public class DataServerConnectionFactory { /** * collection of connections - * key:ip + * key:connectId ip:port */ private final Map MAP = new ConcurrentHashMap<>(); @@ -41,16 +41,16 @@ public class DataServerConnectionFactory { * @param connection */ public void register(Connection connection) { - MAP.put(connection.getRemoteIP(), connection); + MAP.put(getConnectId(connection), connection); } /** - * remove connection by specific ip + * remove connection by specific ip+port * * @param connection */ public void remove(Connection connection) { - MAP.remove(connection.getRemoteIP()); + MAP.remove(getConnectId(connection)); } /** @@ -60,6 +60,10 @@ public void remove(Connection connection) { * @return */ public Connection getConnection(String ip) { - return MAP.get(ip); + return MAP.values().stream().filter(connection -> ip.equals(connection.getRemoteIP()) && connection.isFine()).findFirst().orElse(null); + } + + private String getConnectId(Connection connection) { + return connection.getRemoteIP() + ":" + connection.getRemotePort(); } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerNodeFactory.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerNodeFactory.java index 53a484bea..44976090d 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerNodeFactory.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/DataServerNodeFactory.java @@ -16,12 +16,6 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver; -import com.alipay.remoting.Connection; -import com.alipay.sofa.registry.consistency.hash.ConsistentHash; -import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; -import com.alipay.sofa.registry.server.data.node.DataServerNode; -import com.google.common.collect.Lists; - import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -30,6 +24,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import com.alipay.remoting.Connection; +import com.alipay.sofa.registry.consistency.hash.ConsistentHash; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.node.DataServerNode; +import com.google.common.collect.Lists; + /** * the factory to hold other dataservers and connection connected to them * @@ -59,14 +59,18 @@ public class DataServerNodeFactory { * * @param dataServerNode */ - public static void register(DataServerNode dataServerNode, - DataServerConfig dataServerBootstrapConfig) { + public static void register(DataServerNode dataServerNode, DataServerConfig dataServerConfig) { String dataCenter = dataServerNode.getDataCenter(); - if (!MAP.containsKey(dataCenter)) { - MAP.put(dataCenter, new ConcurrentHashMap<>()); + Map dataMap = MAP.get(dataCenter); + if (dataMap == null) { + Map newMap = new ConcurrentHashMap<>(); + dataMap = MAP.putIfAbsent(dataCenter, newMap); + if (dataMap == null) { + dataMap = newMap; + } } - MAP.get(dataCenter).put(dataServerNode.getIp(), dataServerNode); - refreshConsistent(dataCenter, dataServerBootstrapConfig); + dataMap.put(dataServerNode.getIp(), dataServerNode); + refreshConsistent(dataCenter, dataServerConfig); } /** @@ -74,31 +78,29 @@ public static void register(DataServerNode dataServerNode, * * @param dataCenter */ - public static void refreshConsistent(String dataCenter, - DataServerConfig dataServerBootstrapConfig) { + public static void refreshConsistent(String dataCenter, DataServerConfig dataServerConfig) { List dataServerNodes = Lists.newArrayList(MAP.get(dataCenter).values()); - if (dataServerBootstrapConfig.getLocalDataCenter().equals(dataCenter)) { + if (dataServerConfig.getLocalDataCenter().equals(dataCenter)) { if (!MAP.get(dataCenter).keySet().contains(DataServerConfig.IP)) { - dataServerNodes.add(new DataServerNode(DataServerConfig.IP, - dataServerBootstrapConfig.getLocalDataCenter(), null)); + dataServerNodes.add(new DataServerNode(DataServerConfig.IP, dataServerConfig + .getLocalDataCenter(), null)); } } CONSISTENT_HASH_MAP.put(dataCenter, - new ConsistentHash<>(dataServerBootstrapConfig.getNumberOfReplicas(), dataServerNodes)); + new ConsistentHash<>(dataServerConfig.getNumberOfReplicas(), dataServerNodes)); } /** * for single node consistentHash - * @param dataServerBootstrapConfig + * @param dataServerConfig */ - public static void initConsistent(DataServerConfig dataServerBootstrapConfig) { + public static void initConsistent(DataServerConfig dataServerConfig) { if (init.compareAndSet(false, true)) { List dataServerNodes = Lists.newArrayList(); - dataServerNodes.add(new DataServerNode(DataServerConfig.IP, dataServerBootstrapConfig + dataServerNodes.add(new DataServerNode(DataServerConfig.IP, dataServerConfig .getLocalDataCenter(), null)); - CONSISTENT_HASH_MAP.put(dataServerBootstrapConfig.getLocalDataCenter(), - new ConsistentHash<>(dataServerBootstrapConfig.getNumberOfReplicas(), - dataServerNodes)); + CONSISTENT_HASH_MAP.put(dataServerConfig.getLocalDataCenter(), new ConsistentHash<>( + dataServerConfig.getNumberOfReplicas(), dataServerNodes)); } } @@ -155,8 +157,7 @@ public static Set getIps(String dataCenter) { * @param dataCenter * @param ip */ - public static void remove(String dataCenter, String ip, - DataServerConfig dataServerBootstrapConfig) { + public static void remove(String dataCenter, String ip, DataServerConfig dataServerConfig) { if (MAP.containsKey(dataCenter)) { Map map = MAP.get(dataCenter); if (map != null) { @@ -168,7 +169,7 @@ public static void remove(String dataCenter, String ip, map.remove(ip); } } - refreshConsistent(dataCenter, dataServerBootstrapConfig); + refreshConsistent(dataCenter, dataServerConfig); } /** @@ -193,7 +194,7 @@ public static void remove(String dataCenter) { public static DataServerNode computeDataServerNode(String dataCenter, String dataInfoId) { ConsistentHash consistentHash = CONSISTENT_HASH_MAP.get(dataCenter); if (consistentHash != null) { - return CONSISTENT_HASH_MAP.get(dataCenter).getNodeFor(dataInfoId); + return consistentHash.getNodeFor(dataInfoId); } return null; } @@ -202,18 +203,9 @@ public static List computeDataServerNodes(String dataCenter, Str int backupNodes) { ConsistentHash consistentHash = CONSISTENT_HASH_MAP.get(dataCenter); if (consistentHash != null) { - return CONSISTENT_HASH_MAP.get(dataCenter).getNUniqueNodesFor(dataInfoId, backupNodes); + return consistentHash.getNUniqueNodesFor(dataInfoId, backupNodes); } return null; } - /** - * get all datacenters - * - * @return - */ - public static Set getAllDataCenters() { - return MAP.keySet(); - } - } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/GetSyncDataHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/GetSyncDataHandler.java index c315110ee..62456108e 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/GetSyncDataHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/GetSyncDataHandler.java @@ -25,6 +25,8 @@ import com.alipay.sofa.registry.server.data.remoting.DataNodeExchanger; import org.springframework.beans.factory.annotation.Autowired; +import java.util.concurrent.Executor; + /** * * @author qian.lqlq @@ -71,6 +73,11 @@ public void onCallback(Channel channel, Object message) { public void onException(Channel channel, Throwable exception) { callback.onException(exception); } + + @Override + public Executor getExecutor() { + return callback.getExecutor(); + } }; } }); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/SyncDataCallback.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/SyncDataCallback.java index a5785f314..5f673218c 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/SyncDataCallback.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/SyncDataCallback.java @@ -16,24 +16,23 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver; +import java.util.Collection; +import java.util.concurrent.Executor; + +import org.springframework.util.CollectionUtils; + import com.alipay.remoting.Connection; import com.alipay.remoting.InvokeCallback; import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.SyncData; import com.alipay.sofa.registry.common.model.dataserver.SyncDataRequest; -import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.data.change.DataChangeTypeEnum; import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; -import org.springframework.util.CollectionUtils; - -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.Executor; /** * @@ -44,7 +43,7 @@ public class SyncDataCallback implements InvokeCallback { private static final Logger LOGGER = LoggerFactory.getLogger(SyncDataCallback.class); - private static final Executor EXECUTOR = ExecutorFactory.newFixedThreadPool(20, + private static final Executor EXECUTOR = ExecutorFactory.newFixedThreadPool(5, SyncDataCallback.class.getSimpleName()); private static final int RETRY_COUNT = 3; @@ -96,7 +95,7 @@ public void onResponse(Object obj) { datum.setDataInfoId(syncData.getDataInfoId()); datum.setDataCenter(syncData.getDataCenter()); } - processDatum(datum); + Datum.internDatum(datum); dataChangeEventCenter.sync(DataChangeTypeEnum.COVER, dataSourceTypeEnum, datum); break; } @@ -105,7 +104,7 @@ public void onResponse(Object obj) { if (!CollectionUtils.isEmpty(datums)) { for (Datum datum : datums) { if (datum != null) { - processDatum(datum); + Datum.internDatum(datum); dataChangeEventCenter.sync(DataChangeTypeEnum.MERGE, dataSourceTypeEnum, datum); } @@ -117,16 +116,6 @@ public void onResponse(Object obj) { } } - private void processDatum(Datum datum) { - if (datum != null) { - Map publisherMap = datum.getPubMap(); - - if (publisherMap != null && !publisherMap.isEmpty()) { - publisherMap.forEach((registerId, publisher) -> Publisher.processPublisher(publisher)); - } - } - } - @Override public void onException(Throwable e) { GenericResponse genericResponse = new GenericResponse(); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/FetchDataHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/FetchDataHandler.java index 76c9c0b33..0ebcab135 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/FetchDataHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/FetchDataHandler.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver.handler; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.dataserver.Datum; @@ -25,8 +29,6 @@ import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; import com.alipay.sofa.registry.util.ParaCheckUtil; -import java.util.Map; - /** * processor to get specific data * @@ -35,6 +37,9 @@ */ public class FetchDataHandler extends AbstractClientHandler { + @Autowired + private DatumCache datumCache; + @Override public void checkParam(GetDataRequest request) throws RuntimeException { ParaCheckUtil.checkNotBlank(request.getDataInfoId(), "GetDataRequest.dataInfoId"); @@ -42,7 +47,7 @@ public void checkParam(GetDataRequest request) throws RuntimeException { @Override public Object doHandle(Channel channel, GetDataRequest request) { - return new GenericResponse>().fillSucceed(DatumCache + return new GenericResponse>().fillSucceed(datumCache .getDatumGroupByDataCenter(request.getDataCenter(), request.getDataInfoId())); } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyDataSyncHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyDataSyncHandler.java index 5938070bb..2ae447b81 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyDataSyncHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyDataSyncHandler.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver.handler; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; @@ -29,42 +38,52 @@ import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; +import com.alipay.sofa.registry.server.data.node.DataNodeStatus; import com.alipay.sofa.registry.server.data.remoting.dataserver.GetSyncDataHandler; import com.alipay.sofa.registry.server.data.remoting.dataserver.SyncDataCallback; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; -import com.alipay.sofa.registry.server.data.util.ThreadPoolExecutorDataServer; +import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; +import com.alipay.sofa.registry.server.data.util.DataMetricsThreadPoolExecutor; import com.alipay.sofa.registry.util.NamedThreadFactory; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * * @author qian.lqlq * @version $Id: NotifyDataSyncProcessor.java, v 0.1 2018-03-06 20:04 qian.lqlq Exp $ */ -public class NotifyDataSyncHandler extends AbstractClientHandler { +public class NotifyDataSyncHandler extends AbstractClientHandler implements + AfterWorkingProcess { + + private static final Logger LOGGER = LoggerFactory + .getLogger(NotifyDataSyncHandler.class); - private static final Logger LOGGER = LoggerFactory.getLogger(NotifyDataSyncHandler.class); + @Autowired + private DataServerConfig dataServerConfig; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private GetSyncDataHandler getSyncDataHandler; @Autowired - private GetSyncDataHandler getSyncDataHandler; + private DataChangeEventCenter dataChangeEventCenter; + + private Executor executor = ExecutorFactory + .newFixedThreadPool( + 10, + NotifyDataSyncHandler.class + .getSimpleName()); + + private ThreadPoolExecutor notifyExecutor; @Autowired - private DataChangeEventCenter dataChangeEventCenter; + private DataNodeStatus dataNodeStatus; - private Executor executor = ExecutorFactory.newFixedThreadPool(10, - NotifyDataSyncHandler.class.getSimpleName()); + @Autowired + private DatumCache datumCache; - private ThreadPoolExecutor notifyExecutor; + private static final BlockingQueue noWorkQueue = new LinkedBlockingQueue<>(); @Override public void checkParam(NotifyDataSyncRequest request) throws RuntimeException { @@ -74,25 +93,59 @@ public void checkParam(NotifyDataSyncRequest request) throws RuntimeException { @Override public Object doHandle(Channel channel, NotifyDataSyncRequest request) { final Connection connection = ((BoltChannel) channel).getConnection(); + if (dataNodeStatus.getStatus() != LocalServerStatusEnum.WORKING) { + LOGGER.info("receive notifyDataSync request,but data server not working!"); + noWorkQueue.add(new SyncDataRequestForWorking(connection, request)); + return CommonResponse.buildSuccessResponse(); + } + executorRequest(connection, request); + return CommonResponse.buildSuccessResponse(); + } + + private void executorRequest(Connection connection, NotifyDataSyncRequest request) { executor.execute(() -> { - String dataInfoId = request.getDataInfoId(); - String dataCenter = request.getDataCenter(); - Datum datum = DatumCache.get(dataCenter, dataInfoId); - Long version = (datum == null) ? null : datum.getVersion(); - Long requestVersion = request.getVersion(); - if (version == null || requestVersion == 0L || version < requestVersion) { - LOGGER.info( - "[NotifyDataSyncProcessor] begin get sync data, currentVersion={},request={}", version, - request); - getSyncDataHandler - .syncData(new SyncDataCallback(getSyncDataHandler, connection, new SyncDataRequest(dataInfoId, - dataCenter, version, request.getDataSourceType()), dataChangeEventCenter)); - } else { - LOGGER.info( - "[NotifyDataSyncHandler] not need to sync data, version={}", version); - } + fetchSyncData(connection, request); }); - return CommonResponse.buildSuccessResponse(); + } + + protected void fetchSyncData(Connection connection, NotifyDataSyncRequest request) { + String dataInfoId = request.getDataInfoId(); + String dataCenter = request.getDataCenter(); + Datum datum = datumCache.get(dataCenter, dataInfoId); + Long version = (datum == null) ? null : datum.getVersion(); + Long requestVersion = request.getVersion(); + + if (version == null || requestVersion == 0L || version < requestVersion) { + LOGGER.info( + "[NotifyDataSyncProcessor] begin get sync data, currentVersion={},request={}", + version, request); + getSyncDataHandler.syncData(new SyncDataCallback(getSyncDataHandler, connection, + new SyncDataRequest(dataInfoId, dataCenter, version, request.getDataSourceType()), + dataChangeEventCenter)); + } else { + LOGGER.info( + "[NotifyDataSyncHandler] not need to sync data, currentVersion={},request={}", + version, request); + } + } + + @Override + public void afterWorkingProcess() { + try { + while (!noWorkQueue.isEmpty()) { + SyncDataRequestForWorking event = noWorkQueue.poll(1, TimeUnit.SECONDS); + if (event != null) { + executorRequest(event.getConnection(), event.getRequest()); + } + } + } catch (InterruptedException e) { + LOGGER.error("receive disconnect event after working interrupted!", e); + } + } + + @Override + public int getOrder() { + return 1; } @Override @@ -108,12 +161,11 @@ public Class interest() { @Override public Executor getExecutor() { if (notifyExecutor == null) { - notifyExecutor = new ThreadPoolExecutorDataServer("NotifyDataSyncProcessorExecutor", - dataServerBootstrapConfig.getNotifyDataSyncExecutorMinPoolSize(), - dataServerBootstrapConfig.getNotifyDataSyncExecutorMaxPoolSize(), - dataServerBootstrapConfig.getNotifyDataSyncExecutorKeepAliveTime(), - TimeUnit.SECONDS, new ArrayBlockingQueue<>( - dataServerBootstrapConfig.getNotifyDataSyncExecutorQueueSize()), + notifyExecutor = new DataMetricsThreadPoolExecutor("NotifyDataSyncProcessorExecutor", + dataServerConfig.getNotifyDataSyncExecutorMinPoolSize(), + dataServerConfig.getNotifyDataSyncExecutorMaxPoolSize(), + dataServerConfig.getNotifyDataSyncExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(dataServerConfig.getNotifyDataSyncExecutorQueueSize()), new NamedThreadFactory("DataServer-NotifyDataSyncProcessor-executor", true)); } return notifyExecutor; @@ -128,4 +180,35 @@ public HandlerType getType() { protected Node.NodeType getConnectNodeType() { return Node.NodeType.DATA; } + + private class SyncDataRequestForWorking { + + private final Connection connection; + + private final NotifyDataSyncRequest request; + + public SyncDataRequestForWorking(Connection connection, NotifyDataSyncRequest request) { + this.connection = connection; + this.request = request; + } + + /** + * Getter method for property connection. + * + * @return property value of connection + */ + public Connection getConnection() { + return connection; + } + + /** + * Getter method for property request. + * + * @return property value of request + */ + public NotifyDataSyncRequest getRequest() { + return request; + } + + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyFetchDatumHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyFetchDatumHandler.java index cb11bf076..068b57e35 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyFetchDatumHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/handler/NotifyFetchDatumHandler.java @@ -16,6 +16,11 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver.handler; +import java.util.Map; +import java.util.Map.Entry; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.GenericResponse; @@ -37,12 +42,9 @@ import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerConnectionFactory; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; +import com.alipay.sofa.registry.server.data.renew.LocalDataServerCleanHandler; import com.alipay.sofa.registry.server.data.util.TimeUtil; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Map; -import java.util.Map.Entry; /** * @@ -63,14 +65,17 @@ public class NotifyFetchDatumHandler extends AbstractServerHandler> versionMap = request.getDataVersionMap(); long version = request.getChangeVersion(); String ip = request.getIp(); if (version >= dataServerCache.getCurVersion()) { if (versionMap.isEmpty()) { - LOGGER - .info( - "[NotifyFetchDatumHandler] get changeVersion map is empty,change version is {},current version is {},ip is {}", - version, dataServerCache.getCurVersion(), ip); + LOGGER.info( + "[NotifyFetchDatumHandler] get changeVersion map is empty,change version is {},current version is {},ip is {}", + version, dataServerCache.getCurVersion(), ip); dataServerCache.synced(version, ip); } else { ExecutorFactory.getCommonExecutor().execute(() -> { @@ -97,7 +105,7 @@ public Object doHandle(Channel channel, NotifyFetchDatumRequest request) { Map map = dataCenterEntry.getValue(); for (Entry dataInfoEntry : map.entrySet()) { String dataInfoId = dataInfoEntry.getKey(); - Datum datum = DatumCache.get(dataCenter, dataInfoId); + Datum datum = datumCache.get(dataCenter, dataInfoId); if (datum != null) { long inVersion = dataInfoEntry.getValue(); long currentVersion = datum.getVersion(); @@ -121,8 +129,7 @@ public Object doHandle(Channel channel, NotifyFetchDatumRequest request) { }); } } else { - LOGGER.info( - "[NotifyFetchDatumHandler] ignore notify because changeVersion {} is less than {},ip is {}", + LOGGER.info("[NotifyFetchDatumHandler] ignore notify because changeVersion {} is less than {},ip is {}", version, dataServerCache.getCurVersion(), ip); } return CommonResponse.buildSuccessResponse(); @@ -136,23 +143,25 @@ public Object doHandle(Channel channel, NotifyFetchDatumRequest request) { * @param dataInfoId */ private void fetchDatum(String targetIp, String dataCenter, String dataInfoId) { - while ((dataServerCache.getDataServers(dataServerConfig.getLocalDataCenter()).keySet()) - .contains(targetIp)) { + while (dataServerConnectionFactory.getConnection(targetIp) != null) { Connection connection = dataServerConnectionFactory.getConnection(targetIp); if (connection == null || !connection.isFine()) { throw new RuntimeException(String.format("connection of %s is not available", targetIp)); } try { - Server syncServer = boltExchange.getServer(dataServerBootstrapConfig - .getSyncDataPort()); + Server syncServer = boltExchange.getServer(dataServerConfig.getSyncDataPort()); GenericResponse> response = (GenericResponse>) syncServer .sendSync(syncServer.getChannel(connection.getRemoteAddress()), new GetDataRequest(dataInfoId, dataCenter), - dataServerBootstrapConfig.getRpcTimeout()); + dataServerConfig.getRpcTimeout()); if (response.isSuccess()) { Datum datum = response.getData().get(dataCenter); + if (datum != null) { + // wrap by WordCache + datum = Datum.internDatum(datum); + dataChangeEventCenter.sync(DataChangeTypeEnum.COVER, DataSourceTypeEnum.BACKUP, datum); LOGGER diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/AbstractTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/AbstractTask.java index 64f80fc16..39c7d0488 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/AbstractTask.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/AbstractTask.java @@ -18,6 +18,7 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import java.util.concurrent.TimeUnit; @@ -74,4 +75,9 @@ public void run() { */ public abstract TimeUnit getTimeUnit(); + /** + * get type to match post + * @return + */ + public abstract StartTaskTypeEnum getStartTaskTypeEnum(); } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ConnectionRefreshTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ConnectionRefreshTask.java index 6a19451ca..9e1b96586 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ConnectionRefreshTask.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ConnectionRefreshTask.java @@ -18,7 +18,9 @@ import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent; +import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent.FromType; import com.alipay.sofa.registry.server.data.event.EventCenter; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import org.springframework.beans.factory.annotation.Autowired; @@ -41,7 +43,8 @@ public class ConnectionRefreshTask extends AbstractTask { public void handle() { DataServerChangeItem dataServerChangeItem = metaServerService.getDateServers(); if (dataServerChangeItem != null) { - eventCenter.post(new DataServerChangeEvent(dataServerChangeItem)); + eventCenter + .post(new DataServerChangeEvent(dataServerChangeItem, FromType.CONNECT_TASK)); } } @@ -60,4 +63,9 @@ public TimeUnit getTimeUnit() { return TimeUnit.SECONDS; } + @Override + public StartTaskTypeEnum getStartTaskTypeEnum() { + return StartTaskTypeEnum.CONNECT_DATA; + } + } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/LogMetricsTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/LogMetricsTask.java new file mode 100644 index 000000000..facf38493 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/LogMetricsTask.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.dataserver.task; + +import com.alipay.remoting.ProtocolCode; +import com.alipay.remoting.ProtocolManager; +import com.alipay.remoting.rpc.protocol.RpcProtocol; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.metrics.TaskMetrics; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class LogMetricsTask extends AbstractTask { + private TaskMetrics taskMetrics = TaskMetrics.getInstance(); + + @Autowired + private DataServerConfig dataServerConfig; + + private static final Logger EXE_LOGGER = LoggerFactory.getLogger("DATA-PROFILE-DIGEST", + "[ExecutorMetrics]"); + + public LogMetricsTask() { + ThreadPoolExecutor boltDefaultExecutor = (ThreadPoolExecutor) ProtocolManager + .getProtocol(ProtocolCode.fromBytes(RpcProtocol.PROTOCOL_CODE)).getCommandHandler() + .getDefaultExecutor(); + taskMetrics.registerThreadExecutor("Data-BoltDefaultExecutor", boltDefaultExecutor); + } + + public void printExecutorMetrics() { + EXE_LOGGER.info(taskMetrics.metricsString()); + } + + @Override + public void handle() { + printExecutorMetrics(); + } + + @Override + public int getDelay() { + return dataServerConfig.getLogMetricsFixedDelay(); + } + + @Override + public int getInitialDelay() { + return dataServerConfig.getLogMetricsFixedDelay(); + } + + @Override + public TimeUnit getTimeUnit() { + return TimeUnit.SECONDS; + } + + @Override + public StartTaskTypeEnum getStartTaskTypeEnum() { + return StartTaskTypeEnum.LOG_METRICS; + } +} diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ReNewNodeTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/RenewNodeTask.java similarity index 80% rename from server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ReNewNodeTask.java rename to server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/RenewNodeTask.java index 6fda74933..965fc9f40 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/ReNewNodeTask.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/dataserver/task/RenewNodeTask.java @@ -16,6 +16,7 @@ */ package com.alipay.sofa.registry.server.data.remoting.dataserver.task; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import org.springframework.beans.factory.annotation.Autowired; @@ -24,16 +25,16 @@ /** * * @author shangyu.wh - * @version $Id: ReNewNodeTask.java, v 0.1 2018-04-02 20:56 shangyu.wh Exp $ + * @version $Id: RenewNodeTask.java, v 0.1 2018-04-02 20:56 shangyu.wh Exp $ */ -public class ReNewNodeTask extends AbstractTask { +public class RenewNodeTask extends AbstractTask { @Autowired private IMetaServerService metaServerService; @Override public void handle() { - metaServerService.reNewNodeTask(); + metaServerService.renewNodeTask(); } @Override @@ -50,4 +51,9 @@ public int getInitialDelay() { public TimeUnit getTimeUnit() { return TimeUnit.SECONDS; } + + @Override + public StartTaskTypeEnum getStartTaskTypeEnum() { + return StartTaskTypeEnum.RENEW; + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractClientHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractClientHandler.java index 8f48d5166..41a4735d4 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractClientHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractClientHandler.java @@ -21,6 +21,10 @@ import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; /** * @@ -29,7 +33,13 @@ */ public abstract class AbstractClientHandler implements ChannelHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractClientHandler.class); + @Autowired + private ThreadPoolExecutor defaultRequestExecutor; + + private static final Logger LOGGER = LoggerFactory + .getLogger(AbstractClientHandler.class); + + private static final Logger LOGGER_EXCHANGE = LoggerFactory.getLogger("DATA-EXCHANGE"); @Override public void connected(Channel channel) { @@ -64,7 +74,7 @@ public void received(Channel channel, T message) { @Override public Object reply(Channel channel, T request) { try { - logRequest(request); + logRequest(channel, request); checkParam(request); return doHandle(channel, request); } catch (Exception e) { @@ -102,8 +112,13 @@ public Object reply(Channel channel, T request) { * * @param request */ - protected void logRequest(T request) { - log(request.toString()); + protected void logRequest(Channel channel, T request) { + if (channel != null) { + log(new StringBuilder("Remote:").append(channel.getRemoteAddress()).append(" Request:") + .append(request).toString()); + } else { + log(request.toString()); + } } /** @@ -112,7 +127,7 @@ protected void logRequest(T request) { * @param log */ protected void log(String log) { - LOGGER.info(new StringBuilder("[").append(getClassName()).append("] ").append(log) + LOGGER_EXCHANGE.info(new StringBuilder("[").append(getClassName()).append("] ").append(log) .toString()); } @@ -129,4 +144,9 @@ public Class interest() { private String getClassName() { return this.getClass().getSimpleName(); } + + @Override + public Executor getExecutor() { + return defaultRequestExecutor; + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractServerHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractServerHandler.java index 34630ed39..5ec302942 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractServerHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/handler/AbstractServerHandler.java @@ -22,6 +22,10 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.RemotingException; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; /** * @@ -30,23 +34,31 @@ */ public abstract class AbstractServerHandler implements ChannelHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractServerHandler.class); + @Autowired + private ThreadPoolExecutor defaultRequestExecutor; + + private static final Logger LOGGER = LoggerFactory + .getLogger(AbstractServerHandler.class); + + private static final Logger LOGGER_CONNECT = LoggerFactory.getLogger("DATA-CONNECT"); + + private static final Logger LOGGER_EXCHANGE = LoggerFactory.getLogger("DATA-EXCHANGE"); @Override public void connected(Channel channel) throws RemotingException { if (channel != null && channel.isConnected()) { - LOGGER - .info(getConnectNodeType() + " node connected,remote address:" - + channel.getRemoteAddress() + " localAddress:" + channel.getLocalAddress()); + LOGGER_CONNECT.info(getConnectNodeType() + " node connected,remote address:" + + channel.getRemoteAddress() + " localAddress:" + + channel.getLocalAddress()); } } @Override public void disconnected(Channel channel) throws RemotingException { if (channel != null && !channel.isConnected()) { - LOGGER - .info(getConnectNodeType() + " node disconnected,remote address:" - + channel.getRemoteAddress() + " localAddress:" + channel.getLocalAddress()); + LOGGER_CONNECT.info(getConnectNodeType() + " node disconnected,remote address:" + + channel.getRemoteAddress() + " localAddress:" + + channel.getLocalAddress()); } } @@ -112,7 +124,6 @@ public Class interest() { * @param request */ protected void logRequest(Channel channel, T request) { - log(request.toString()); if (channel != null) { log(new StringBuilder("Remote:").append(channel.getRemoteAddress()).append(" Request:") .append(request).toString()); @@ -127,7 +138,7 @@ protected void logRequest(Channel channel, T request) { * @param log */ protected void log(String log) { - LOGGER.info(new StringBuilder("[").append(getClassName()).append("] ").append(log) + LOGGER_EXCHANGE.info(new StringBuilder("[").append(getClassName()).append("] ").append(log) .toString()); } @@ -139,4 +150,9 @@ protected void log(String log) { private String getClassName() { return this.getClass().getSimpleName(); } + + @Override + public Executor getExecutor() { + return defaultRequestExecutor; + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/DefaultMetaServiceImpl.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/DefaultMetaServiceImpl.java index dd9ecdc49..0b90b8064 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/DefaultMetaServiceImpl.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/DefaultMetaServiceImpl.java @@ -16,37 +16,45 @@ */ package com.alipay.sofa.registry.server.data.remoting.metaserver; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.metaserver.DataNode; +import com.alipay.sofa.registry.common.model.metaserver.FetchProvideDataRequest; import com.alipay.sofa.registry.common.model.metaserver.GetNodesRequest; import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import com.alipay.sofa.registry.common.model.metaserver.ReNewNodesRequest; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.common.model.metaserver.RenewNodesRequest; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.jraft.bootstrap.RaftClient; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.bolt.BoltChannel; import com.alipay.sofa.registry.remoting.exchange.message.Request; +import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DataServerCache; import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; import com.alipay.sofa.registry.server.data.node.DataServerNode; import com.alipay.sofa.registry.server.data.remoting.MetaNodeExchanger; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; /** * @@ -59,7 +67,10 @@ public class DefaultMetaServiceImpl implements IMetaServerService { .getLogger(DefaultMetaServiceImpl.class); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; + + @Autowired + private ThreadPoolExecutor defaultRequestExecutor; @Autowired private MetaNodeExchanger metaNodeExchanger; @@ -67,6 +78,9 @@ public class DefaultMetaServiceImpl implements IMetaServerService { @Autowired private MetaServerConnectionFactory metaServerConnectionFactory; + @Autowired + private DataServerCache dataServerCache; + private RaftClient raftClient; private AtomicBoolean clientStart = new AtomicBoolean(false); @@ -74,18 +88,25 @@ public class DefaultMetaServiceImpl implements IMetaServerService { @Override public Map> getMetaServerMap() { HashMap> map = new HashMap<>(); - Set set = dataServerBootstrapConfig.getMetaServerIpAddresses(); + Set set = dataServerConfig.getMetaServerIpAddresses(); Map connectionMap = metaServerConnectionFactory - .getConnections(dataServerBootstrapConfig.getLocalDataCenter()); + .getConnections(dataServerConfig.getLocalDataCenter()); Connection connection = null; try { if (connectionMap.isEmpty()) { - connection = ((BoltChannel) metaNodeExchanger.connect(new URL( - set.iterator().next(), dataServerBootstrapConfig.getMetaServerPort()))) - .getConnection(); + List list = new ArrayList(set); + Collections.shuffle(list); + connection = ((BoltChannel) metaNodeExchanger.connect(new URL(list.iterator() + .next(), dataServerConfig.getMetaServerPort()))).getConnection(); } else { - connection = connectionMap.values().iterator().next(); + List connections = new ArrayList<>(connectionMap.values()); + Collections.shuffle(connections); + connection = connections.iterator().next(); + if (!connection.isFine()) { + connection = ((BoltChannel) metaNodeExchanger.connect(new URL(connection + .getRemoteIP(), dataServerConfig.getMetaServerPort()))).getConnection(); + } } GetNodesRequest request = new GetNodesRequest(NodeType.META); @@ -106,11 +127,10 @@ public URL getRequestUrl() { Map> metaNodesMap = result.getNodes(); if (metaNodesMap != null && !metaNodesMap.isEmpty()) { - Map metaNodeMap = metaNodesMap.get(dataServerBootstrapConfig + Map metaNodeMap = metaNodesMap.get(dataServerConfig .getLocalDataCenter()); if (metaNodeMap != null && !metaNodeMap.isEmpty()) { - map.put(dataServerBootstrapConfig.getLocalDataCenter(), - metaNodeMap.keySet()); + map.put(dataServerConfig.getLocalDataCenter(), metaNodeMap.keySet()); } else { LOGGER .error( @@ -127,6 +147,8 @@ public URL getRequestUrl() { .warn( "[DefaultMetaServiceImpl] refresh connections from metaServer error,refresh leader : {}", con); + throw new RuntimeException( + "[DefaultMetaServiceImpl] refresh connections from metaServer error!", e); } return map; } @@ -139,13 +161,18 @@ public DataServerNode getDataServer(String dataCenter, String dataInfoId) { @Override public List getDataServers(String dataCenter, String dataInfoId) { return DataServerNodeFactory.computeDataServerNodes(dataCenter, dataInfoId, - dataServerBootstrapConfig.getStoreNodes()); + dataServerConfig.getStoreNodes()); + } + + @Override + public Collection getDataServers(String dataCenter) { + return DataServerNodeFactory.getDataServerNodes(dataCenter).values(); } @Override public DataServerChangeItem getDateServers() { Map connectionMap = metaServerConnectionFactory - .getConnections(dataServerBootstrapConfig.getLocalDataCenter()); + .getConnections(dataServerConfig.getLocalDataCenter()); String leader = getLeader().getIp(); if (connectionMap.containsKey(leader)) { Connection connection = connectionMap.get(leader); @@ -191,15 +218,15 @@ public URL getRequestUrl() { @Override public List getOtherDataCenters() { - Set all = new HashSet<>(DataServerNodeFactory.getAllDataCenters()); - all.remove(dataServerBootstrapConfig.getLocalDataCenter()); + Set all = new HashSet<>(dataServerCache.getAllDataCenters()); + all.remove(dataServerConfig.getLocalDataCenter()); return new ArrayList<>(all); } @Override - public void reNewNodeTask() { + public void renewNodeTask() { Map connectionMap = metaServerConnectionFactory - .getConnections(dataServerBootstrapConfig.getLocalDataCenter()); + .getConnections(dataServerConfig.getLocalDataCenter()); for (Entry connectEntry : connectionMap.entrySet()) { String ip = connectEntry.getKey(); //just send to leader @@ -207,13 +234,13 @@ public void reNewNodeTask() { Connection connection = connectEntry.getValue(); if (connection.isFine()) { try { - ReNewNodesRequest reNewNodesRequest = new ReNewNodesRequest<>( + RenewNodesRequest renewNodesRequest = new RenewNodesRequest<>( new DataNode(new URL(DataServerConfig.IP), - dataServerBootstrapConfig.getLocalDataCenter())); + dataServerConfig.getLocalDataCenter())); metaNodeExchanger.request(new Request() { @Override public Object getRequestBody() { - return reNewNodesRequest; + return renewNodesRequest; } @Override @@ -222,12 +249,12 @@ public URL getRequestUrl() { } }).getResult(); } catch (Exception e) { - LOGGER.error("[ReNewNodeTask] reNew data node to metaServer error : {}", + LOGGER.error("[RenewNodeTask] renew data node to metaServer error : {}", ip, e); String newip = refreshLeader().getIp(); LOGGER .warn( - "[ReNewNodeTask] reNew data node to metaServer error,leader refresh: {}", + "[RenewNodeTask] renew data node to metaServer error,leader refresh: {}", newip); } } else { @@ -241,12 +268,59 @@ public URL getRequestUrl() { } } + @Override + public ProvideData fetchData(String dataInfoId) { + + Map connectionMap = metaServerConnectionFactory + .getConnections(dataServerConfig.getLocalDataCenter()); + String leader = getLeader().getIp(); + if (connectionMap.containsKey(leader)) { + Connection connection = connectionMap.get(leader); + if (connection.isFine()) { + try { + Request request = new Request() { + + @Override + public FetchProvideDataRequest getRequestBody() { + + return new FetchProvideDataRequest(dataInfoId); + } + + @Override + public URL getRequestUrl() { + return new URL(connection.getRemoteIP(), connection.getRemotePort()); + } + }; + + Response response = metaNodeExchanger.request(request); + + Object result = response.getResult(); + if (result instanceof ProvideData) { + return (ProvideData) result; + } else { + LOGGER.error("fetch null provider data!"); + throw new RuntimeException("MetaNodeService fetch null provider data!"); + } + } catch (Exception e) { + LOGGER.error("fetch provider data error! " + e.getMessage(), e); + throw new RuntimeException("fetch provider data error! " + e.getMessage(), e); + } + } + } + String newip = refreshLeader().getIp(); + LOGGER.warn( + "[ConnectionRefreshTask] refresh connections metaServer not fine,refresh leader : {}", + newip); + return null; + + } + @Override public void startRaftClient() { try { if (clientStart.compareAndSet(false, true)) { String serverConf = getServerConfig(); - raftClient = new RaftClient(getGroup(), serverConf); + raftClient = new RaftClient(getGroup(), serverConf, defaultRequestExecutor); raftClient.start(); } } catch (Exception e) { @@ -257,10 +331,10 @@ public void startRaftClient() { private String getServerConfig() { String ret = ""; - Set ips = dataServerBootstrapConfig.getMetaServerIpAddresses(); + Set ips = dataServerConfig.getMetaServerIpAddresses(); if (ips != null && !ips.isEmpty()) { ret = ips.stream().map(ip -> ip + ":" + ValueConstants.RAFT_SERVER_PORT) - .collect(Collectors.joining(",")); + .collect(Collectors.joining(",")); } if (ret.isEmpty()) { throw new IllegalArgumentException("Init raft server config error!"); @@ -269,8 +343,7 @@ private String getServerConfig() { } private String getGroup() { - return ValueConstants.RAFT_SERVER_GROUP + "_" - + dataServerBootstrapConfig.getLocalDataCenter(); + return ValueConstants.RAFT_SERVER_GROUP + "_" + dataServerConfig.getLocalDataCenter(); } @Override diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/IMetaServerService.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/IMetaServerService.java index e98f8e453..e6621f341 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/IMetaServerService.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/IMetaServerService.java @@ -16,14 +16,16 @@ */ package com.alipay.sofa.registry.server.data.remoting.metaserver; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; -import com.alipay.sofa.registry.server.data.node.DataServerNode; - +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.server.data.cache.DataServerChangeItem; +import com.alipay.sofa.registry.server.data.node.DataServerNode; + /** * The interface Meta server service. * @author qian.lqlq @@ -56,6 +58,14 @@ public interface IMetaServerService { */ List getDataServers(String dataCenter, String dataInfoId); + /** + * Gets data servers. + * + * @param dataCenter the data center + * @return the data servers + */ + Collection getDataServers(String dataCenter); + /** * Gets date servers. * @@ -73,7 +83,7 @@ public interface IMetaServerService { /** * scheduler update data server expireTime */ - void reNewNodeTask(); + void renewNodeTask(); /** * start raft client for get leader send request @@ -91,4 +101,11 @@ public interface IMetaServerService { * @return */ PeerId refreshLeader(); + + /** + * get provider data + * @param dataInfoId + * @return + */ + ProvideData fetchData(String dataInfoId); } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/NotifyProvideDataChangeHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/NotifyProvideDataChangeHandler.java new file mode 100644 index 000000000..d6ea6b2a9 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/NotifyProvideDataChangeHandler.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.metaserver.handler; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.Node.NodeType; +import com.alipay.sofa.registry.common.model.metaserver.DataOperator; +import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; +import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; +import com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.ProvideDataProcessor; + +/** + * + * @author shangyu.wh + * @version $Id: DataChangeRequestHandler.java, v 0.1 2017-12-12 15:09 shangyu.wh Exp $ + */ +public class NotifyProvideDataChangeHandler extends AbstractClientHandler { + + private static final Logger LOGGER = LoggerFactory + .getLogger(NotifyProvideDataChangeHandler.class); + + @Autowired + private IMetaServerService metaServerService; + + @Autowired + private ProvideDataProcessor provideDataProcessorManager; + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + protected NodeType getConnectNodeType() { + return NodeType.DATA; + } + + @Override + public void checkParam(Object request) throws RuntimeException { + + } + + @Override + public Object doHandle(Channel channel, Object request) { + LOGGER.info("Received notifyProvideDataChange: {}", request); + + NotifyProvideDataChange notifyProvideDataChange = (NotifyProvideDataChange) request; + String dataInfoId = notifyProvideDataChange.getDataInfoId(); + if (notifyProvideDataChange.getDataOperator() != DataOperator.REMOVE) { + ProvideData provideData = metaServerService.fetchData(dataInfoId); + provideDataProcessorManager.changeDataProcess(provideData); + } + return null; + } + + @Override + public Class interest() { + return NotifyProvideDataChange.class; + } + + @Override + public CommonResponse buildFailedResponse(String msg) { + return CommonResponse.buildFailedResponse(msg); + } + +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/ServerChangeHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/ServerChangeHandler.java index 0174f5776..66f18f9ea 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/ServerChangeHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/handler/ServerChangeHandler.java @@ -16,23 +16,27 @@ */ package com.alipay.sofa.registry.server.data.remoting.metaserver.handler; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent; +import com.alipay.sofa.registry.server.data.event.DataServerChangeEvent.FromType; import com.alipay.sofa.registry.server.data.event.EventCenter; import com.alipay.sofa.registry.server.data.event.MetaServerChangeEvent; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractClientHandler; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; /** * @@ -41,11 +45,13 @@ */ public class ServerChangeHandler extends AbstractClientHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(ServerChangeHandler.class); + @Autowired - private EventCenter eventCenter; + private EventCenter eventCenter; @Autowired - private DataServerConfig dataServerConfig; + private DataServerConfig dataServerConfig; @Override public void checkParam(NodeChangeResult request) throws RuntimeException { @@ -54,10 +60,11 @@ public void checkParam(NodeChangeResult request) throws RuntimeException { @Override public Object doHandle(Channel channel, NodeChangeResult request) { + LOGGER.info("Received NodeChangeResult: {}", request); ExecutorFactory.getCommonExecutor().execute(() -> { if (request.getNodeType() == NodeType.DATA) { eventCenter.post(new DataServerChangeEvent(request.getNodes(), - request.getDataCenterListVersions())); + request.getDataCenterListVersions(), FromType.META_NOTIFY)); } else if (request.getNodeType() == NodeType.META) { Map> metaNodesMap = request.getNodes(); if (metaNodesMap != null && !metaNodesMap.isEmpty()) { diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessor.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessor.java new file mode 100644 index 000000000..dc08d1fe6 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessor.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.metaserver.provideData; + +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; + +/** + * + * @author kezhu.wukz + * @version 1.0: ProvideDataProcessor.java, v 0.1 2019-12-25 17:26 kezhu.wukz Exp $ + */ +public interface ProvideDataProcessor { + + void changeDataProcess(ProvideData provideData); + + boolean support(ProvideData provideData); +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessorManager.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessorManager.java new file mode 100644 index 000000000..24346f4d9 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/ProvideDataProcessorManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.metaserver.provideData; + +import java.util.ArrayList; +import java.util.Collection; + +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; + +/** + * + * @author kezhu.wukz + * @version 1.0: ProvideDataProcessorManager.java, v 0.1 2019-12-25 17:39 kezhu.wukz Exp $ + */ +public class ProvideDataProcessorManager implements ProvideDataProcessor { + + private Collection provideDataProcessors = new ArrayList<>(); + + public void addProvideDataProcessor(ProvideDataProcessor provideDataProcessor) { + provideDataProcessors.add(provideDataProcessor); + } + + @Override + public void changeDataProcess(ProvideData provideData) { + for (ProvideDataProcessor provideDataProcessor : provideDataProcessors) { + if (provideDataProcessor.support(provideData)) { + provideDataProcessor.changeDataProcess(provideData); + } + } + } + + @Override + public boolean support(ProvideData provideData) { + return false; + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/processor/DatumExpireProvideDataProcessor.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/processor/DatumExpireProvideDataProcessor.java new file mode 100644 index 000000000..ceba5865e --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/provideData/processor/DatumExpireProvideDataProcessor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.processor; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.remoting.metaserver.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; + +/** + * + * @author kezhu.wukz + * @version 1.0: DatumExpireProvideDataProcessor.java, v 0.1 2019-12-26 20:30 kezhu.wukz Exp $ + */ +public class DatumExpireProvideDataProcessor implements ProvideDataProcessor { + + private static final Logger LOGGER = LoggerFactory + .getLogger(DatumExpireProvideDataProcessor.class); + + @Autowired + private DatumLeaseManager datumLeaseManager; + + @Override + public void changeDataProcess(ProvideData provideData) { + if (checkInvalid(provideData)) { + return; + } + boolean enableDataDatumExpire = Boolean.parseBoolean((String) provideData.getProvideData() + .getObject()); + LOGGER.info("Fetch enableDataDatumExpire {} success!", enableDataDatumExpire); + datumLeaseManager.setRenewEnable(enableDataDatumExpire); + } + + private boolean checkInvalid(ProvideData provideData) { + boolean invalid = provideData == null || provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null; + if (invalid) { + LOGGER.warn("Fetch enableDataDatumExpire return invalid data, provideData: {}", + provideData); + } + return invalid; + } + + @Override + public boolean support(ProvideData provideData) { + return ValueConstants.ENABLE_DATA_DATUM_EXPIRE.equals(provideData.getDataInfoId()); + } +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/task/ConnectionRefreshMetaTask.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/task/ConnectionRefreshMetaTask.java index a36967ba0..50946a93a 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/task/ConnectionRefreshMetaTask.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/metaserver/task/ConnectionRefreshMetaTask.java @@ -18,6 +18,7 @@ import com.alipay.sofa.registry.server.data.event.EventCenter; import com.alipay.sofa.registry.server.data.event.MetaServerChangeEvent; +import com.alipay.sofa.registry.server.data.event.StartTaskTypeEnum; import com.alipay.sofa.registry.server.data.remoting.dataserver.task.AbstractTask; import com.alipay.sofa.registry.server.data.remoting.metaserver.IMetaServerService; import org.springframework.beans.factory.annotation.Autowired; @@ -56,4 +57,9 @@ public int getInitialDelay() { public TimeUnit getTimeUnit() { return TimeUnit.SECONDS; } + + @Override + public StartTaskTypeEnum getStartTaskTypeEnum() { + return StartTaskTypeEnum.CONNECT_META; + } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/SessionServerConnectionFactory.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/SessionServerConnectionFactory.java index e8072e5ca..b3384d4dd 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/SessionServerConnectionFactory.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/SessionServerConnectionFactory.java @@ -16,101 +16,120 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.Connection; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.DisconnectEventHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.SessionServerDisconnectEvent; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** - * the factory to hold sesseionserver connections + * the factory to hold SessionServer connections * * @author qian.lqlq + * @author kezhu.wukz * @version $Id: SessionServerConnectionFactory.java, v 0.1 2017-12-06 15:48 qian.lqlq Exp $ */ public class SessionServerConnectionFactory { - private static final Logger LOGGER = LoggerFactory - .getLogger(SessionServerConnectionFactory.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(SessionServerConnectionFactory.class); - private static final int DELAY = 30 * 1000; + private static final Map EMPTY_MAP = new HashMap(0); /** - * collection of connections - * key : processId - * value : connection + * key : SessionServer address + * value: SessionServer processId */ - private final Map MAP = new ConcurrentHashMap<>(); + private final Map SESSION_CONN_PROCESS_ID_MAP = new ConcurrentHashMap<>(); /** - * key : sessionserver host - * value: sesseionserver processId + * key : SessionServer processId + * value: ip:port of clients */ - private final Map PROCESSID_MAP = new ConcurrentHashMap<>(); + private final Map> PROCESS_ID_CONNECT_ID_MAP = new ConcurrentHashMap<>(); /** - * key : sessionserver processId - * value: ip:port of clients + * key : SessionServer processId + * value: pair(SessionServer address, SessionServer connection) */ - private final Map> PROCESSID_CLIENT_MAP = new ConcurrentHashMap<>(); + private final Map PROCESS_ID_SESSION_CONN_MAP = new ConcurrentHashMap<>(); @Autowired private DisconnectEventHandler disconnectEventHandler; + @Autowired + private DataServerConfig dataServerConfig; + /** * register connection * * @param processId - * @param clientHosts + * @param connectIds * @param connection */ - public void register(String processId, Set clientHosts, Connection connection) { - String serverHost = NetUtil.toAddressString(connection.getRemoteAddress()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("session({}, processId={}) registered", serverHost, processId); + public void registerSession(String processId, Set connectIds, Connection connection) { + if (!connection.isFine()) { + return; } - MAP.put(processId, new Pair(serverHost, connection)); - Set ret = PROCESSID_CLIENT_MAP.getOrDefault(processId, null); - if (ret == null) { - PROCESSID_CLIENT_MAP.putIfAbsent(processId, new HashSet<>()); - } - PROCESSID_CLIENT_MAP.get(processId).addAll(clientHosts); - PROCESSID_MAP.put(serverHost, processId); + String sessionConnAddress = NetUtil.toAddressString(connection.getRemoteAddress()); + LOGGER.info("session({}, processId={}) registered", sessionConnAddress, processId); + + SESSION_CONN_PROCESS_ID_MAP.put(sessionConnAddress, processId); + + Set connectIdSet = PROCESS_ID_CONNECT_ID_MAP + .computeIfAbsent(processId, k -> ConcurrentHashMap.newKeySet()); + connectIdSet.addAll(connectIds); + + Pair pair = PROCESS_ID_SESSION_CONN_MAP.computeIfAbsent(processId, k -> new Pair(new ConcurrentHashMap<>())); + pair.getConnections().put(sessionConnAddress, connection); + } /** * * @param processId - * @param clientAddress + * @param connectId */ - public void registerClient(String processId, String clientAddress) { - Set ret = PROCESSID_CLIENT_MAP.getOrDefault(processId, null); - if (ret == null) { - PROCESSID_CLIENT_MAP.putIfAbsent(processId, new HashSet<>()); - } - PROCESSID_CLIENT_MAP.get(processId).add(clientAddress); + public void registerConnectId(String processId, String connectId) { + Set connectIdSet = PROCESS_ID_CONNECT_ID_MAP + .computeIfAbsent(processId, k -> ConcurrentHashMap.newKeySet()); + connectIdSet.add(connectId); } /** - * remove connection by specific host + * session disconnected, The SessionServerDisconnectEvent is triggered only when the last connections is removed */ - public void removeProcess(String sessionServerHost) { - String processId = PROCESSID_MAP.remove(sessionServerHost); + public void sessionDisconnected(String sessionConnAddress) { + String processId = SESSION_CONN_PROCESS_ID_MAP.remove(sessionConnAddress); if (LOGGER.isInfoEnabled()) { - LOGGER.info("session({}, processId={}) unregistered", sessionServerHost, processId); + LOGGER.info("session({}, processId={}) unregistered", sessionConnAddress, processId); } if (processId != null) { - disconnectEventHandler.receive(new SessionServerDisconnectEvent(processId, - sessionServerHost, DELAY)); + Pair pair = PROCESS_ID_SESSION_CONN_MAP.get(processId); + + // remove connection + if (pair != null) { + pair.getConnections().remove(sessionConnAddress); + pair.lastDisconnectedSession = sessionConnAddress; + } + + // The SessionServerDisconnectEvent is triggered only when the last connection is broken + if (pair == null || pair.getConnections().isEmpty()) { + disconnectEventHandler.receive(new SessionServerDisconnectEvent(processId, + sessionConnAddress, dataServerConfig.getSessionDisconnectDelayMs())); + } } } @@ -118,68 +137,67 @@ public void removeProcess(String sessionServerHost) { * * @param processId */ - public Set removeClients(String processId) { - return PROCESSID_CLIENT_MAP.remove(processId); + public Set removeConnectIds(String processId) { + return PROCESS_ID_CONNECT_ID_MAP.remove(processId); } /** - * - * @param processId - * @return + * If the number of connections is 0, and lastDisconnectedSession matched, he ProcessId can be deleted */ - public boolean removeProcessIfMatch(String processId, String sessionServerHost) { - return MAP.remove(processId, new Pair(sessionServerHost, null)); + public boolean removeProcessIfMatch(String processId, String sessionConnAddress) { + Pair emptyPair = new Pair(EMPTY_MAP); + emptyPair.lastDisconnectedSession = sessionConnAddress; + return PROCESS_ID_SESSION_CONN_MAP.remove(processId, emptyPair); } /** - * get all connections - * - * @return + * get connections of SessionServer ( Randomly select a connection for each session ) */ - public List getConnections() { - return MAP.size() <= 0 ? - Collections.EMPTY_LIST : - MAP.values().stream().map(Pair::getConnection).collect(Collectors.toList()); + public List getSessionConnections() { + List list = new ArrayList<>(PROCESS_ID_SESSION_CONN_MAP.size()); + Collection pairs = PROCESS_ID_SESSION_CONN_MAP.values(); + if (pairs != null) { + for (Pair pair : pairs) { + Object[] conns = pair.getConnections().values().toArray(); + if (conns.length > 0) { + int n = pair.roundRobin.incrementAndGet(); + if (n < 0) { + pair.roundRobin.compareAndSet(n, 0); + n = (n == Integer.MIN_VALUE) ? 0 : Math.abs(n); + } + n = n % conns.length; + list.add((Connection) conns[n]); + } + } + } + return list; } /** - * convenient class to store sessionServerHost and connection + * convenient class to store sessionConnAddress and connection */ private static class Pair { - private String sessionServerHost; - private Connection connection; + private AtomicInteger roundRobin = new AtomicInteger(-1); + private Map connections; + private String lastDisconnectedSession; - private Pair(String sessionServerHost, Connection connection) { - this.sessionServerHost = sessionServerHost; - this.connection = connection; + private Pair(Map connections) { + this.connections = connections; } @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Pair pair = (Pair) o; - - return sessionServerHost.equals(pair.sessionServerHost); - } - - @Override - public int hashCode() { - return sessionServerHost.hashCode(); + return connections.equals(((Pair) o).getConnections()) + && (((Pair) o).lastDisconnectedSession.equals(lastDisconnectedSession)); } /** - * Getter method for property connection. + * Getter method for property connections. * - * @return property value of connection + * @return property value of connections */ - private Connection getConnection() { - return connection; + private Map getConnections() { + return connections; } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/ClientDisconnectEvent.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/ClientDisconnectEvent.java index 3659db591..c3f233133 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/ClientDisconnectEvent.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/ClientDisconnectEvent.java @@ -23,16 +23,16 @@ */ public class ClientDisconnectEvent extends DisconnectEvent { - private String host; + private String connectId; /** * constructor - * @param host + * @param connectId * @param registerTimestamp * @param timeoutMs */ - public ClientDisconnectEvent(String host, long registerTimestamp, int timeoutMs) { - this.host = host; + public ClientDisconnectEvent(String connectId, long registerTimestamp, int timeoutMs) { + this.connectId = connectId; setRegisterTimestamp(registerTimestamp); setGmtOccur(System.currentTimeMillis()); setTimeoutMs(timeoutMs); @@ -40,12 +40,12 @@ public ClientDisconnectEvent(String host, long registerTimestamp, int timeoutMs) } /** - * Getter method for property host. + * Getter method for property connectId. * - * @return property value of host + * @return property value of connectId */ - public String getHost() { - return host; + public String getConnectId() { + return connectId; } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/DisconnectEventHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/DisconnectEventHandler.java index f262b420f..8d5333d26 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/DisconnectEventHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/disconnect/DisconnectEventHandler.java @@ -16,42 +16,59 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.change.event.ClientChangeEvent; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; +import com.alipay.sofa.registry.server.data.node.DataNodeStatus; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Set; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Executor; +import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; /** * @author qian.lqlq * @version $Id: ClientDisconnectEventHandler.java, v 0.1 2017-12-07 15:32 qian.lqlq Exp $ */ -public class DisconnectEventHandler implements InitializingBean { +public class DisconnectEventHandler implements InitializingBean, AfterWorkingProcess { + + private static final Logger LOGGER = LoggerFactory + .getLogger(DisconnectEventHandler.class); - private static final Logger LOGGER = LoggerFactory - .getLogger(DisconnectEventHandler.class); + private static final Logger LOGGER_START = LoggerFactory + .getLogger("DATA-START-LOGS"); /** * a DelayQueue that contains client disconnect events */ - private final DelayQueue EVENT_QUEUE = new DelayQueue<>(); + private final DelayQueue EVENT_QUEUE = new DelayQueue<>(); @Autowired - private SessionServerConnectionFactory sessionServerConnectionFactory; + private SessionServerConnectionFactory sessionServerConnectionFactory; @Autowired - private DataChangeEventCenter dataChangeEventCenter; + private DataChangeEventCenter dataChangeEventCenter; @Autowired - private DataServerConfig dataServerConfig; + private DataServerConfig dataServerConfig; + + @Autowired + private DataNodeStatus dataNodeStatus; + + private static final int BLOCK_FOR_ALL_SYNC = 5000; + + private static final BlockingQueue noWorkQueue = new LinkedBlockingQueue<>(); /** * receive disconnect event of client @@ -64,13 +81,47 @@ public void receive(DisconnectEvent event) { LOGGER.info("receive session off event: sessionServerHost={}, processId={}", sessionServerDisconnectEvent.getSessionServerHost(), sessionServerDisconnectEvent.getProcessId()); + } else if (event.getType() == DisconnectTypeEnum.CLIENT) { + ClientDisconnectEvent clientDisconnectEvent = (ClientDisconnectEvent) event; + LOGGER.info("receive client off event: connectId={}", + clientDisconnectEvent.getConnectId()); + } + + if (dataNodeStatus.getStatus() != LocalServerStatusEnum.WORKING) { + LOGGER.info("receive disconnect event,but data server not working!"); + noWorkQueue.add(event); + return; } EVENT_QUEUE.add(event); } + public void afterWorkingProcess() { + try { + /* + * After the snapshot data is synchronized during startup, it is queued and then placed asynchronously into + * DatumCache. When the notification becomes WORKING, there may be data in the queue that is not executed + * to DatumCache. So it need to sleep for a while. + */ + TimeUnit.MILLISECONDS.sleep(BLOCK_FOR_ALL_SYNC); + + while (!noWorkQueue.isEmpty()) { + DisconnectEvent event = noWorkQueue.poll(1, TimeUnit.SECONDS); + if (event != null) { + receive(event); + } + } + } catch (InterruptedException e) { + LOGGER.error("receive disconnect event after working interrupted!", e); + } + } + + @Override + public int getOrder() { + return 0; + } + @Override public void afterPropertiesSet() { - LOGGER.info("begin start DisconnectEventHandler"); Executor executor = ExecutorFactory .newSingleThreadExecutor(DisconnectEventHandler.class.getSimpleName()); executor.execute(() -> { @@ -78,22 +129,22 @@ public void afterPropertiesSet() { try { DisconnectEvent disconnectEvent = EVENT_QUEUE.take(); if (disconnectEvent.getType() == DisconnectTypeEnum.SESSION_SERVER) { - SessionServerDisconnectEvent event = (SessionServerDisconnectEvent) disconnectEvent; - String processId = event.getProcessId(); + SessionServerDisconnectEvent event = (SessionServerDisconnectEvent) disconnectEvent; + String processId = event.getProcessId(); //check processId confirm remove,and not be registered again when delay time String sessionServerHost = event.getSessionServerHost(); if (sessionServerConnectionFactory - .removeProcessIfMatch(processId, sessionServerHost)) { - Set clientHosts = sessionServerConnectionFactory - .removeClients(processId); + .removeProcessIfMatch(processId,sessionServerHost)) { + Set connectIds = sessionServerConnectionFactory + .removeConnectIds(processId); - LOGGER.info("session off is triggered: sessionServerHost={}, clientHost={}, processId={}", + LOGGER.info("session off is triggered: sessionServerHost={}, connectId={}, processId={}", sessionServerHost, - clientHosts, processId); + connectIds, processId); - if (clientHosts != null && !clientHosts.isEmpty()) { - for (String host : clientHosts) { - unPub(host, event.getRegisterTimestamp()); + if (connectIds != null && !connectIds.isEmpty()) { + for (String connectId : connectIds) { + unPub(connectId, event.getRegisterTimestamp()); } } } else { @@ -102,23 +153,23 @@ public void afterPropertiesSet() { } } else { ClientDisconnectEvent event = (ClientDisconnectEvent) disconnectEvent; - unPub(event.getHost(), event.getRegisterTimestamp()); + unPub(event.getConnectId(), event.getRegisterTimestamp()); } } catch (Throwable e) { LOGGER.error("handle client disconnect event failed", e); } } }); - LOGGER.info("start DisconnectEventHandler success"); + LOGGER_START.info("start DisconnectEventHandler success"); } /** * - * @param host + * @param connectId * @param registerTimestamp */ - private void unPub(String host, long registerTimestamp) { - dataChangeEventCenter.onChange(new ClientChangeEvent(host, dataServerConfig + private void unPub(String connectId, long registerTimestamp) { + dataChangeEventCenter.onChange(new ClientChangeEvent(connectId, dataServerConfig .getLocalDataCenter(), registerTimestamp)); } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardService.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardService.java index 43931d46c..b896521fc 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardService.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardService.java @@ -28,10 +28,9 @@ public interface ForwardService { /** * Need forward boolean. * - * @param dataInfoId the data info id * @return the boolean */ - boolean needForward(String dataInfoId); + boolean needForward(); /** * Forward request object. diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardServiceImpl.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardServiceImpl.java index 7228bc03d..b8985153a 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardServiceImpl.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/forward/ForwardServiceImpl.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.forward; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.remoting.exception.RemotingException; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; @@ -28,9 +32,6 @@ import com.alipay.sofa.registry.server.data.remoting.DataNodeExchanger; import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.List; /** * The type Forward service. @@ -43,7 +44,7 @@ public class ForwardServiceImpl implements ForwardService { private static final Logger LOGGER = LoggerFactory.getLogger(ForwardServiceImpl.class); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private DataNodeStatus dataNodeStatus; @@ -52,10 +53,10 @@ public class ForwardServiceImpl implements ForwardService { private DataNodeExchanger dataNodeExchanger; /** - * @see ForwardService#needForward(String) + * @see ForwardService#needForward() */ @Override - public boolean needForward(String dataInfoId) { + public boolean needForward() { return dataNodeStatus.getStatus() != LocalServerStatusEnum.WORKING; } @@ -66,9 +67,9 @@ public boolean needForward(String dataInfoId) { public Object forwardRequest(String dataInfoId, Object request) throws RemotingException { try { // 1. get store nodes - List dataServerNodes = DataServerNodeFactory.computeDataServerNodes( - dataServerBootstrapConfig.getLocalDataCenter(), dataInfoId, - dataServerBootstrapConfig.getStoreNodes()); + List dataServerNodes = DataServerNodeFactory + .computeDataServerNodes(dataServerConfig.getLocalDataCenter(), dataInfoId, + dataServerConfig.getStoreNodes()); // 2. find next node if (null == dataServerNodes || dataServerNodes.size() <= 0) { diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/ClientOffHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/ClientOffHandler.java index 2ebd53d7f..5df47ec9d 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/ClientOffHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/ClientOffHandler.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.dataserver.ClientOffRequest; @@ -24,10 +28,8 @@ import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.ClientDisconnectEvent; import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.DisconnectEventHandler; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.List; /** * processor to remove data of specific clients immediately @@ -38,11 +40,14 @@ public class ClientOffHandler extends AbstractServerHandler { @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private DisconnectEventHandler disconnectEventHandler; + @Autowired + private DatumLeaseManager datumLeaseManager; + @Override public void checkParam(ClientOffRequest request) throws RuntimeException { ParaCheckUtil.checkNotEmpty(request.getHosts(), "ClientOffRequest.hosts"); @@ -52,8 +57,11 @@ public void checkParam(ClientOffRequest request) throws RuntimeException { public Object doHandle(Channel channel, ClientOffRequest request) { List hosts = request.getHosts(); for (String host : hosts) { + // stop task of this connectId + datumLeaseManager.remove(host); + disconnectEventHandler.receive(new ClientDisconnectEvent(host, request.getGmtOccur(), - dataServerBootstrapConfig.getClientOffDelayMs() * 10)); + dataServerConfig.getClientOffDelayMs())); } return CommonResponse.buildSuccessResponse(); } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DataServerConnectionHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DataServerConnectionHandler.java index 6f41e1f95..02316aee8 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DataServerConnectionHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DataServerConnectionHandler.java @@ -16,16 +16,22 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.RemotingException; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; -import org.springframework.beans.factory.annotation.Autowired; /** + * + * Connection handler for session connect data server + * + * fix shangyu.wh * @author xuanbei * @since 2019/2/15 */ @@ -46,13 +52,13 @@ public void connected(Channel channel) throws RemotingException { @Override public void disconnected(Channel channel) throws RemotingException { super.disconnected(channel); - sessionServerConnectionFactory.removeProcess(NetUtil.toAddressString(channel + sessionServerConnectionFactory.sessionDisconnected(NetUtil.toAddressString(channel .getRemoteAddress())); } @Override protected Node.NodeType getConnectNodeType() { - return Node.NodeType.DATA; + return NodeType.SESSION; } @Override diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DatumSnapshotHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DatumSnapshotHandler.java new file mode 100644 index 000000000..6b281aff1 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/DatumSnapshotHandler.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.PublisherDigestUtil; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; +import com.alipay.sofa.registry.server.data.change.event.DatumSnapshotEvent; +import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; +import com.alipay.sofa.registry.util.ParaCheckUtil; + +/** + * handling snapshot request + * + * @author kezhu.wukz + * @version $Id: ClientOffProcessor.java, v 0.1 2019-05-30 15:48 kezhu.wukz Exp $ + */ +public class DatumSnapshotHandler extends AbstractServerHandler { + + private static final Logger RENEW_LOGGER = LoggerFactory.getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[DatumSnapshotHandler]"); + + /** Limited List Printing */ + private static final int LIMITED_LIST_SIZE_FOR_PRINT = 10; + + @Autowired + private DataChangeEventCenter dataChangeEventCenter; + + @Autowired + private DatumLeaseManager datumLeaseManager; + + @Autowired + private DatumCache datumCache; + + @Autowired + private ThreadPoolExecutor renewDatumProcessorExecutor; + + @Override + public Executor getExecutor() { + return renewDatumProcessorExecutor; + } + + @Override + public void checkParam(DatumSnapshotRequest request) throws RuntimeException { + ParaCheckUtil.checkNotBlank(request.getConnectId(), "DatumSnapshotRequest.connectId"); + ParaCheckUtil.checkNotEmpty(request.getPublishers(), "DatumSnapshotRequest.publishers"); + } + + @Override + public Object doHandle(Channel channel, DatumSnapshotRequest request) { + RENEW_LOGGER.info("Received datumSnapshotRequest: {}", request); + + String connectId = WordCache.getInstance().getWordCache(request.getConnectId()); + + // convert to pubMap, and wrap it by WordCache + Map pubMap = new HashMap<>(); + List publishers = request.getPublishers(); + if (publishers != null) { + for (Publisher publisher : publishers) { + Publisher.internPublisher(publisher); + pubMap.put(publisher.getRegisterId(), publisher); + } + } + + // diff the cache and snapshot + boolean isDiff = true; + Map cachePubMap = datumCache.getOwnByConnectId(connectId); + if (cachePubMap == null) { + RENEW_LOGGER + .info( + ">>>>>>> connectId={}, cachePubMap.size=0, pubMap.size={}, isDiff={}, the diff is: pubMap={}", + connectId, pubMap.size(), isDiff, limitedToString(pubMap.values())); + } else { + List diffPub1 = subtract(pubMap, cachePubMap); + List diffPub2 = subtract(cachePubMap, pubMap); + if (diffPub1.size() == 0 && diffPub2.size() == 0) { + isDiff = false; + } + RENEW_LOGGER + .info( + ">>>>>>> connectId={}, cachePubMap.size={}, pubMap.size={}, isDiff={}, the diff is: pubMap-cachePubMap=(size:{}){}, cachePubMap-pubMap=(size:{}){}", + connectId, cachePubMap.size(), pubMap.size(), isDiff, diffPub1.size(), + limitedToString(diffPub1), diffPub2.size(), limitedToString(diffPub2)); + } + + if (isDiff) { + // build DatumSnapshotEvent and send to eventCenter + dataChangeEventCenter.onChange(new DatumSnapshotEvent(connectId, cachePubMap, pubMap)); + } + + // record the renew timestamp + datumLeaseManager.renew(connectId); + + return CommonResponse.buildSuccessResponse(); + } + + /** + * Limited List Printing + */ + private String limitedToString(Collection publishers) { + Iterator it = publishers.iterator(); + if (!it.hasNext()) + return "[]"; + + StringBuilder sb = new StringBuilder(); + sb.append('['); + int i = 1; + for (;;) { + Publisher e = it.next(); + sb.append("Publisher{dataInfoId='").append(e.getDataInfoId()).append('\''); + sb.append(", cell='").append(e.getCell()).append('\''); + sb.append(", registerId='").append(e.getRegisterId()).append('\''); + sb.append(", version=").append(e.getVersion()); + sb.append(", sourceAddress=").append(e.getSourceAddress()); + sb.append(", registerTimestamp=").append(e.getRegisterTimestamp()); + sb.append(", clientRegisterTimestamp=").append(e.getClientRegisterTimestamp()); + sb.append('}'); + if (!it.hasNext() || i++ >= LIMITED_LIST_SIZE_FOR_PRINT) + return sb.append(']').toString(); + sb.append(',').append(' '); + } + } + + private List subtract(Map pubMap1, Map pubMap2) { + List list = new ArrayList(); + for (Map.Entry entry : pubMap1.entrySet()) { + String registerId = entry.getKey(); + Publisher publisher1 = entry.getValue(); + Publisher publisher2 = pubMap2.get(registerId); + if (publisher2 == null + || PublisherDigestUtil.getDigestValue(publisher1) != PublisherDigestUtil + .getDigestValue(publisher2)) { + list.add(publisher1); + } + } + return list; + } + + @Override + public CommonResponse buildFailedResponse(String msg) { + return CommonResponse.buildFailedResponse(msg); + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Class interest() { + return DatumSnapshotRequest.class; + } + + @Override + protected Node.NodeType getConnectNodeType() { + return Node.NodeType.DATA; + } +} diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataHandler.java index 76682983e..f2c9a9c7f 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataHandler.java @@ -16,6 +16,12 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.dataserver.Datum; @@ -23,20 +29,10 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.forward.ForwardService; -import com.alipay.sofa.registry.server.data.util.ThreadPoolExecutorDataServer; -import com.alipay.sofa.registry.util.NamedThreadFactory; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * processor to get specific data @@ -48,14 +44,21 @@ public class GetDataHandler extends AbstractServerHandler { /** LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(GetDataHandler.class); + @Autowired private ForwardService forwardService; @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DatumCache datumCache; + @Autowired private ThreadPoolExecutor getDataProcessorExecutor; + @Override + public Executor getExecutor() { + return getDataProcessorExecutor; + } + @Override public void checkParam(GetDataRequest request) throws RuntimeException { ParaCheckUtil.checkNotBlank(request.getDataInfoId(), "GetDataRequest.dataInfoId"); @@ -64,7 +67,7 @@ public void checkParam(GetDataRequest request) throws RuntimeException { @Override public Object doHandle(Channel channel, GetDataRequest request) { String dataInfoId = request.getDataInfoId(); - if (forwardService.needForward(dataInfoId)) { + if (forwardService.needForward()) { try { LOGGER.warn("[forward] Get data request forward, request: {}", request); return forwardService.forwardRequest(dataInfoId, request); @@ -77,7 +80,7 @@ public Object doHandle(Channel channel, GetDataRequest request) { } } - return new GenericResponse>().fillSucceed(DatumCache + return new GenericResponse>().fillSucceed(datumCache .getDatumGroupByDataCenter(request.getDataCenter(), dataInfoId)); } @@ -101,16 +104,4 @@ protected Node.NodeType getConnectNodeType() { return Node.NodeType.DATA; } - @Override - public Executor getExecutor() { - if (getDataProcessorExecutor == null) { - getDataProcessorExecutor = new ThreadPoolExecutorDataServer("GetDataProcessorExecutor", - dataServerBootstrapConfig.getGetDataExecutorMinPoolSize(), - dataServerBootstrapConfig.getGetDataExecutorMaxPoolSize(), - dataServerBootstrapConfig.getGetDataExecutorKeepAliveTime(), TimeUnit.SECONDS, - new ArrayBlockingQueue<>(dataServerBootstrapConfig.getGetDataExecutorQueueSize()), - new NamedThreadFactory("DataServer-GetDataProcessor-executor", true)); - } - return getDataProcessorExecutor; - } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataVersionsHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataVersionsHandler.java index e2fa8e4e4..24f58a89b 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataVersionsHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/GetDataVersionsHandler.java @@ -16,21 +16,24 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.Node; -import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.GetDataVersionRequest; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.util.ParaCheckUtil; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - /** * processor to get versions of specific dataInfoIds * @@ -38,6 +41,18 @@ * @version $Id: GetDataVersionsProcessor.java, v 0.1 2017-12-06 19:56 qian.lqlq Exp $ */ public class GetDataVersionsHandler extends AbstractServerHandler { + + @Autowired + private DatumCache datumCache; + + @Autowired + private ThreadPoolExecutor getDataProcessorExecutor; + + @Override + public Executor getExecutor() { + return getDataProcessorExecutor; + } + @Override protected void logRequest(Channel channel, GetDataVersionRequest request) { } @@ -52,17 +67,17 @@ public Object doHandle(Channel channel, GetDataVersionRequest request) { Map> map = new HashMap<>(); List dataInfoIds = request.getDataInfoIds(); for (String dataInfoId : dataInfoIds) { - Map datumMap = DatumCache.get(dataInfoId); - Set> entrySet = datumMap.entrySet(); - for (Entry entry : entrySet) { + Map datumMap = datumCache.getVersions(dataInfoId); + Set> entrySet = datumMap.entrySet(); + for (Entry entry : entrySet) { String dataCenter = entry.getKey(); - Datum datum = entry.getValue(); - if (datum != null) { - if (!map.containsKey(dataCenter)) { - map.put(dataCenter, new HashMap<>()); - } - map.get(dataCenter).put(dataInfoId, datum.getVersion()); + Long version = entry.getValue(); + Map dataInfoIdToVersionMap = map.get(dataCenter); + if (dataInfoIdToVersionMap == null) { + dataInfoIdToVersionMap = new HashMap<>(dataInfoIds.size()); + map.put(dataCenter, dataInfoIdToVersionMap); } + dataInfoIdToVersionMap.put(dataInfoId, version); } } return new GenericResponse>>().fillSucceed(map); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/PublishDataHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/PublishDataHandler.java index 94caad81e..36c74a464 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/PublishDataHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/PublishDataHandler.java @@ -16,11 +16,18 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.PublishType; import com.alipay.sofa.registry.common.model.dataserver.PublishDataRequest; import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; @@ -29,15 +36,8 @@ import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; import com.alipay.sofa.registry.server.data.remoting.sessionserver.forward.ForwardService; -import com.alipay.sofa.registry.server.data.util.ThreadPoolExecutorDataServer; -import com.alipay.sofa.registry.util.NamedThreadFactory; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** * processor to publish data @@ -63,17 +63,11 @@ public class PublishDataHandler extends AbstractServerHandler(dataServerConfig.getPublishExecutorQueueSize()), - new NamedThreadFactory("DataServer-PublishProcessorExecutor-executor", true)); - } + @Autowired + private ThreadPoolExecutor publishProcessorExecutor; @Override public void checkParam(PublishDataRequest request) throws RuntimeException { @@ -93,8 +87,8 @@ public void checkParam(PublishDataRequest request) throws RuntimeException { @Override public Object doHandle(Channel channel, PublishDataRequest request) { - Publisher publisher = Publisher.processPublisher(request.getPublisher()); - if (forwardService.needForward(publisher.getDataInfoId())) { + Publisher publisher = Publisher.internPublisher(request.getPublisher()); + if (forwardService.needForward()) { LOGGER.warn("[forward] Publish request refused, request: {}", request); CommonResponse response = new CommonResponse(); response.setSuccess(false); @@ -103,9 +97,15 @@ public Object doHandle(Channel channel, PublishDataRequest request) { } dataChangeEventCenter.onChange(publisher, dataServerConfig.getLocalDataCenter()); + if (publisher.getPublishType() != PublishType.TEMPORARY) { - sessionServerConnectionFactory.registerClient(request.getSessionServerProcessId(), - publisher.getSourceAddress().getAddressString()); + String connectId = WordCache.getInstance().getWordCache( + publisher.getSourceAddress().getAddressString() + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString()); + sessionServerConnectionFactory.registerConnectId(request.getSessionServerProcessId(), + connectId); + // record the renew timestamp + datumLeaseManager.renew(connectId); } return CommonResponse.buildSuccessResponse(); @@ -133,6 +133,6 @@ protected Node.NodeType getConnectNodeType() { @Override public Executor getExecutor() { - return publishExecutor; + return publishProcessorExecutor; } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/RenewDatumHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/RenewDatumHandler.java new file mode 100644 index 000000000..7d3cea30b --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/RenewDatumHandler.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; + +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.GenericResponse; +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.PublisherDigestUtil; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; +import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; +import com.alipay.sofa.registry.util.ParaCheckUtil; + +/** + * handling snapshot request + * + * @author kezhu.wukz + * @version $Id: RenewDatumHandler.java, v 0.1 2019-05-30 15:48 kezhu.wukz Exp $ + */ +public class RenewDatumHandler extends AbstractServerHandler implements + AfterWorkingProcess { + + /** LOGGER */ + private static final Logger LOGGER = LoggerFactory.getLogger(RenewDatumHandler.class); + + private static final Logger RENEW_LOGGER = LoggerFactory.getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[RenewDatumHandler]"); + + private final AtomicBoolean renewEnabled = new AtomicBoolean(false); + + @Autowired + private DatumLeaseManager datumLeaseManager; + + @Autowired + private DatumCache datumCache; + + @Autowired + private ThreadPoolExecutor renewDatumProcessorExecutor; + + @Autowired + private DataServerConfig dataServerConfig; + + @Override + public Executor getExecutor() { + return renewDatumProcessorExecutor; + } + + @Override + protected void logRequest(Channel channel, RenewDatumRequest request) { + } + + @Override + public void checkParam(RenewDatumRequest request) throws RuntimeException { + ParaCheckUtil.checkNotBlank(request.getConnectId(), "RenewDatumRequest.connectId"); + ParaCheckUtil.checkNotBlank(request.getDigestSum(), "RenewDatumRequest.digestSum"); + } + + @Override + public Object doHandle(Channel channel, RenewDatumRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doHandle: request={}", request); + } + + if (!renewEnabled.get()) { + LOGGER.warn("Renew request refused, renewEnabled is false, request: {}", request); + GenericResponse response = new GenericResponse(); + response.setSuccess(false); + response.setMessage("Renew request refused, renewEnabled is false yet"); + return response; + } + + boolean theSame = renewDatum(request); + + return new GenericResponse().fillSucceed(theSame); + } + + @Override + public CommonResponse buildFailedResponse(String msg) { + return CommonResponse.buildFailedResponse(msg); + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Class interest() { + return RenewDatumRequest.class; + } + + @Override + protected Node.NodeType getConnectNodeType() { + return Node.NodeType.DATA; + } + + /** + * 1. Update the timestamp corresponding to connectId in datumCache + * 2. Compare checksum: Get all pubs corresponding to the connId from datumCache and calculate checksum. + */ + private boolean renewDatum(RenewDatumRequest request) { + String connectId = WordCache.getInstance().getWordCache(request.getConnectId()); + String renewDigest = request.getDigestSum(); + + // Get all pubs corresponding to the connectId from datumCache + Map publisherMap = datumCache.getOwnByConnectId(connectId); + String cacheDigest = null; + if (publisherMap != null && publisherMap.values().size() > 0) { + cacheDigest = String.valueOf(PublisherDigestUtil.getDigestValueSum(publisherMap + .values())); + } + + // record the renew timestamp + datumLeaseManager.renew(connectId); + + boolean result = StringUtils.equals(renewDigest, cacheDigest); + + if (!result) { + RENEW_LOGGER.info("Digest different! renewDatumRequest={}", request); + } + return result; + } + + @Override + public void afterWorkingProcess() { + renewDatumProcessorExecutor.submit(() -> { + /* + * After the snapshot data is synchronized during startup, it is queued and then placed asynchronously into + * DatumCache. When the notification becomes WORKING, there may be data in the queue that is not executed + * to DatumCache. So it need to sleep for a while. + */ + try { + TimeUnit.MILLISECONDS.sleep(dataServerConfig.getRenewEnableDelaySec()); + } catch (InterruptedException e) { + LOGGER.error(e.getMessage(), e); + } + renewEnabled.set(true); + }); + } + + @Override + public int getOrder() { + return 100; + } +} diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/SessionServerRegisterHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/SessionServerRegisterHandler.java index a136fa06f..3f61dba3a 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/SessionServerRegisterHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/SessionServerRegisterHandler.java @@ -16,6 +16,11 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.dataserver.SessionServerRegisterRequest; @@ -24,10 +29,6 @@ import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashSet; -import java.util.Set; /** * @@ -47,11 +48,11 @@ public void checkParam(SessionServerRegisterRequest request) throws RuntimeExcep @Override public Object doHandle(Channel channel, SessionServerRegisterRequest request) { - Set clientHosts = request.getClientHosts(); - if (clientHosts == null) { - clientHosts = new HashSet<>(); + Set connectIds = request.getConnectIds(); + if (connectIds == null) { + connectIds = new HashSet<>(); } - sessionServerConnectionFactory.register(request.getProcessId(), clientHosts, + sessionServerConnectionFactory.registerSession(request.getProcessId(), connectIds, ((BoltChannel) channel).getConnection()); return CommonResponse.buildSuccessResponse(); } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/UnPublishDataHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/UnPublishDataHandler.java index 2a297ed6b..639cda28c 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/UnPublishDataHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/remoting/sessionserver/handler/UnPublishDataHandler.java @@ -16,19 +16,30 @@ */ package com.alipay.sofa.registry.server.data.remoting.sessionserver.handler; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.UnPublishDataRequest; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DatumCache; import com.alipay.sofa.registry.server.data.cache.UnPublisher; import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.data.remoting.sessionserver.forward.ForwardService; +import com.alipay.sofa.registry.server.data.renew.DatumLeaseManager; import com.alipay.sofa.registry.util.ParaCheckUtil; -import org.springframework.beans.factory.annotation.Autowired; /** * processor to unPublish specific data @@ -50,6 +61,20 @@ public class UnPublishDataHandler extends AbstractServerHandler pubMap = datum.getPubMap(); + if (pubMap != null) { + Publisher publisher = pubMap.get(request.getRegisterId()); + if (publisher != null) { + return publisher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString(); + } + } + } + return null; + } + @Override public CommonResponse buildFailedResponse(String msg) { return CommonResponse.buildFailedResponse(msg); diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/DatumLeaseManager.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/DatumLeaseManager.java new file mode 100644 index 000000000..3be35c8a1 --- /dev/null +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/DatumLeaseManager.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.data.renew; + +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + +import org.apache.commons.lang.time.DateFormatUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.event.AfterWorkingProcess; +import com.alipay.sofa.registry.server.data.node.DataNodeStatus; +import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.ClientDisconnectEvent; +import com.alipay.sofa.registry.server.data.remoting.sessionserver.disconnect.DisconnectEventHandler; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * + * @author kezhu.wukz + * @version $Id: DatumExpiredCleaner.java, v 0.1 2019-06-03 21:08 kezhu.wukz Exp $ + */ +public class DatumLeaseManager implements AfterWorkingProcess { + private static final Logger LOGGER = LoggerFactory + .getLogger(DatumLeaseManager.class); + private static final TimeZone TIME_ZONE = TimeZone + .getTimeZone("Asia/Shanghai"); + private static final Logger RENEW_LOGGER = LoggerFactory + .getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[DatumLeaseManager]"); + + /** record the latest heartbeat time for each connectId, format: connectId -> lastRenewTimestamp */ + private final Map connectIdRenewTimestampMap = new ConcurrentHashMap<>(); + + /** lock for connectId , format: connectId -> true */ + private ConcurrentHashMap locksForConnectId = new ConcurrentHashMap(); + + private volatile boolean serverWorking = false; + + private volatile boolean renewEnable = true; + + private AsyncHashedWheelTimer datumAsyncHashedWheelTimer; + + @Autowired + private DataServerConfig dataServerConfig; + + @Autowired + private DisconnectEventHandler disconnectEventHandler; + + @Autowired + private DatumCache datumCache; + + @Autowired + private DataNodeStatus dataNodeStatus; + + private ScheduledThreadPoolExecutor executorForHeartbeatLess; + + private ScheduledFuture futureForHeartbeatLess; + + /** + * constructor + */ + @PostConstruct + public void init() { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setDaemon(true); + datumAsyncHashedWheelTimer = new AsyncHashedWheelTimer(threadFactoryBuilder.setNameFormat( + "Registry-DatumLeaseManager-WheelTimer").build(), 100, TimeUnit.MILLISECONDS, 1024, + dataServerConfig.getDatumLeaseManagerExecutorThreadSize(), + dataServerConfig.getDatumLeaseManagerExecutorQueueSize(), threadFactoryBuilder + .setNameFormat("Registry-DatumLeaseManager-WheelExecutor-%d").build(), + new TaskFailedCallback() { + @Override + public void executionRejected(Throwable e) { + LOGGER.error("executionRejected: " + e.getMessage(), e); + } + + @Override + public void executionFailed(Throwable e) { + LOGGER.error("executionFailed: " + e.getMessage(), e); + } + }); + + executorForHeartbeatLess = new ScheduledThreadPoolExecutor(1, threadFactoryBuilder + .setNameFormat("Registry-DatumLeaseManager-ExecutorForHeartbeatLess").build()); + scheduleEvictTaskForHeartbeatLess(); + } + + /** + * reset EvictTaskForHeartbeatLess + */ + public synchronized void reset() { + LOGGER.info("reset is called, EvictTaskForHeartbeatLess will delay {}s", + dataServerConfig.getDatumTimeToLiveSec()); + if (futureForHeartbeatLess != null) { + futureForHeartbeatLess.cancel(false); + } + scheduleEvictTaskForHeartbeatLess(); + } + + private void scheduleEvictTaskForHeartbeatLess() { + futureForHeartbeatLess = executorForHeartbeatLess.scheduleWithFixedDelay( + new EvictTaskForHeartbeatLess(), dataServerConfig.getDatumTimeToLiveSec(), + dataServerConfig.getDatumTimeToLiveSec(), TimeUnit.SECONDS); + } + + /** + * record the renew timestamp + */ + public void renew(String connectId) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("renew: connectId={}", connectId); + } + + // record the renew timestamp + connectIdRenewTimestampMap.put(connectId, System.currentTimeMillis()); + // try to trigger evict task + scheduleEvictTask(connectId, 0); + } + + /** + * remove connectId if clientOff + */ + public void remove(String connectId) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("remove: connectId={}", connectId); + } + + Long removed = connectIdRenewTimestampMap.remove(connectId); + if (removed != null) { + LOGGER.info("remove connectId({}) because it is clientOff", connectId); + } + } + + /** + * trigger evict task: if connectId expired, create ClientDisconnectEvent to cleanup datums bind to the connectId + * PS: every connectId allows only one task to be created + */ + private void scheduleEvictTask(String connectId, long delaySec) { + delaySec = (delaySec <= 0) ? dataServerConfig.getDatumTimeToLiveSec() : delaySec; + + // lock for connectId: every connectId allows only one task to be created + Boolean ifAbsent = locksForConnectId.putIfAbsent(connectId, true); + if (ifAbsent != null) { + return; + } + + datumAsyncHashedWheelTimer.newTimeout(_timeout -> { + boolean continued = true; + long nextDelaySec = 0; + try { + // release lock + locksForConnectId.remove(connectId); + + // get lastRenewTime of this connectId + Long lastRenewTime = connectIdRenewTimestampMap.get(connectId); + if (lastRenewTime == null) { + // connectId is already clientOff + LOGGER.info("EvictTask(connectId={}) stop because already disconnected", connectId); + return; + } + + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("EvictTask: connectId={}, lastRenewTime={}", connectId, format(lastRenewTime)); + } + + /* + * 1. lastRenewTime expires, then: + * - build ClientOffEvent and hand it to DataChangeEventCenter. + * - It will not be scheduled next time, so terminated. + * 2. lastRenewTime not expires, then: + * - trigger the next schedule + */ + boolean isExpired = + System.currentTimeMillis() - lastRenewTime > dataServerConfig.getDatumTimeToLiveSec() * 1000L; + if (!isRenewEnable()) { + LOGGER.info( + "scheduleEvictTask({}) skipped because isRenewEnable() is false, lastRenewTime is {}, DataNodeStatus is {}, will retry after {}s", + connectId, format(lastRenewTime), dataNodeStatus.getStatus(), + dataServerConfig.getDatumTimeToLiveSec()); + nextDelaySec = dataServerConfig.getDatumTimeToLiveSec(); + } else if (isExpired) { + int ownPubSize = getOwnPubSize(connectId); + if (ownPubSize > 0) { + LOGGER.info("Evict connectId({}) because expired, lastRenewTime is {}, pub.size is {}", + connectId, format(lastRenewTime), ownPubSize); + evict(connectId); + } + connectIdRenewTimestampMap.remove(connectId, lastRenewTime); + continued = false; + } else { + nextDelaySec = dataServerConfig.getDatumTimeToLiveSec() + - (System.currentTimeMillis() - lastRenewTime) / 1000L; + nextDelaySec = nextDelaySec <= 0 ? 1 : nextDelaySec; + } + + } catch (Exception e) { + LOGGER.error("Error in task of datumAsyncHashedWheelTimer", e); + } + if (continued) { + scheduleEvictTask(connectId, nextDelaySec); + } + }, delaySec, TimeUnit.SECONDS); + + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("scheduleEvictTask: connectId={}, delaySec={}", connectId, delaySec); + } + + } + + private int getOwnPubSize(String connectId) { + Map ownPubs = datumCache.getOwnByConnectId(connectId); + return ownPubs != null ? ownPubs.size() : 0; + } + + private void evict(String connectId) { + disconnectEventHandler.receive(new ClientDisconnectEvent(connectId, System + .currentTimeMillis(), 0)); + } + + private String format(long lastRenewTime) { + return DateFormatUtils.format(lastRenewTime, "yyyy-MM-dd HH:mm:ss", TIME_ZONE); + } + + @Override + public void afterWorkingProcess() { + /* + * After the snapshot data is synchronized during startup, it is queued and then placed asynchronously into + * DatumCache. When the notification becomes WORKING, there may be data in the queue that is not executed + * to DatumCache. So it need to sleep for a while. + */ + executorForHeartbeatLess.schedule(() -> { + serverWorking = true; + }, dataServerConfig.getRenewEnableDelaySec(), TimeUnit.SECONDS); + } + + @Override + public int getOrder() { + return 0; + } + + /** + * evict own connectIds with heartbeat less + */ + private class EvictTaskForHeartbeatLess implements Runnable { + + @Override + public void run() { + // If in a non-working state, cannot clean up because the renew request cannot be received at this time. + if (!isRenewEnable()) { + LOGGER + .info( + "EvictTaskForHeartbeatLess skipped because isRenewEnable() is false, DataNodeStatus is {}, will retry after {}s", + dataNodeStatus.getStatus(), dataServerConfig.getDatumTimeToLiveSec()); + return; + } + + Set allConnectIds = datumCache.getAllConnectIds(); + for (String connectId : allConnectIds) { + Long timestamp = connectIdRenewTimestampMap.get(connectId); + // no heartbeat + if (timestamp == null) { + int ownPubSize = getOwnPubSize(connectId); + if (ownPubSize > 0) { + LOGGER.info("Evict connectId({}) because no heartbeat, pub.size is {}", + connectId, ownPubSize); + evict(connectId); + } + } + } + + LOGGER.info("connectIdRenewTimestampMap.size is {}", connectIdRenewTimestampMap.size()); + } + } + + public void setRenewEnable(boolean renewEnable) { + this.renewEnable = renewEnable; + } + + private boolean isRenewEnable() { + return renewEnable && serverWorking; + } + +} \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/correction/LocalDataServerCleanHandler.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/LocalDataServerCleanHandler.java similarity index 85% rename from server/server/data/src/main/java/com/alipay/sofa/registry/server/data/correction/LocalDataServerCleanHandler.java rename to server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/LocalDataServerCleanHandler.java index bfc1fc3c8..4e91f5525 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/correction/LocalDataServerCleanHandler.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/renew/LocalDataServerCleanHandler.java @@ -14,7 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alipay.sofa.registry.server.data.correction; +package com.alipay.sofa.registry.server.data.renew; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.beans.factory.annotation.Autowired; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.metaserver.DataNode; @@ -29,13 +37,6 @@ import com.alipay.sofa.registry.server.data.change.event.DataChangeEventCenter; import com.alipay.sofa.registry.server.data.executor.ExecutorFactory; import com.alipay.sofa.registry.server.data.util.DelayItem; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; /** * @@ -44,11 +45,14 @@ */ public class LocalDataServerCleanHandler { - private static final Logger LOGGER = LoggerFactory - .getLogger(LocalDataServerCleanHandler.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(LocalDataServerCleanHandler.class); + + private static final Logger LOGGER_START = LoggerFactory + .getLogger("DATA-START-LOGS"); @Autowired - private DataServerConfig dataServerBootstrapConfig; + private DataServerConfig dataServerConfig; @Autowired private DataServerCache dataServerCache; @@ -56,20 +60,21 @@ public class LocalDataServerCleanHandler { @Autowired private DataChangeEventCenter dataChangeEventCenter; + @Autowired + private DatumCache datumCache; + private LocalCleanTask task; /** * a DelayQueue that contains clean task */ - private final DelayQueue> EVENT_QUEUE = new DelayQueue<>(); + private final DelayQueue> EVENT_QUEUE = new DelayQueue<>(); /** * constructor */ public LocalDataServerCleanHandler() { - LOGGER.info("[LocalDataServerCleanHandler] begin start LocalDataServerCleanHandler"); - Executor executor = ExecutorFactory - .newSingleThreadExecutor(LocalDataServerCleanHandler.class.getSimpleName()); + Executor executor = ExecutorFactory.newSingleThreadExecutor(LocalDataServerCleanHandler.class.getSimpleName()); executor.execute(() -> { while (true) { try { @@ -81,7 +86,7 @@ public LocalDataServerCleanHandler() { } } }); - LOGGER.info("[LocalDataServerCleanHandler] start LocalDataServerCleanHandler success"); + LOGGER_START.info("[LocalDataServerCleanHandler] start LocalDataServerCleanHandler success"); } /** @@ -89,12 +94,12 @@ public LocalDataServerCleanHandler() { */ public void reset() { synchronized (LocalDataServerCleanHandler.class) { - if (EVENT_QUEUE.isEmpty() && task != null) { + EVENT_QUEUE.clear(); + if (task != null) { task.stop(); - EVENT_QUEUE.clear(); } } - EVENT_QUEUE.add(new DelayItem<>(new LocalCleanTask(), dataServerBootstrapConfig + EVENT_QUEUE.add(new DelayItem<>(new LocalCleanTask(), dataServerConfig .getLocalDataServerCleanDelay())); } @@ -110,16 +115,16 @@ public void run() { try { Map dataNodeMap = dataServerCache - .getDataServers(dataServerBootstrapConfig.getLocalDataCenter()); + .getDataServers(dataServerConfig.getLocalDataCenter()); if (dataNodeMap == null || dataNodeMap.isEmpty()) { LOGGER.warn("Calculate Old BackupTriad,old dataServer list is empty!"); return; } ConsistentHash consistentHash = new ConsistentHash<>( - dataServerBootstrapConfig.getNumberOfReplicas(), dataNodeMap.values()); + dataServerConfig.getNumberOfReplicas(), dataNodeMap.values()); - Map> dataMapAll = DatumCache.getAll(); + Map> dataMapAll = datumCache.getAll(); for (Entry> entryAll : dataMapAll.entrySet()) { String dataCenter = entryAll.getKey(); @@ -137,7 +142,7 @@ public void run() { BackupTriad backupTriad = new BackupTriad(dataInfoId, consistentHash.getNUniqueNodesFor(dataInfoId, - dataServerBootstrapConfig.getStoreNodes())); + dataServerConfig.getStoreNodes())); if (!backupTriad.containsSelf()) { if (datum != null) { int size = datum.getPubMap() != null ? datum.getPubMap().size() @@ -154,7 +159,7 @@ public void run() { } catch (Throwable e) { LOGGER.error("[LocalDataServerCleanHandler] clean local datum task error!", e); } finally { - EVENT_QUEUE.add(new DelayItem<>(new LocalCleanTask(), dataServerBootstrapConfig + EVENT_QUEUE.add(new DelayItem<>(new LocalCleanTask(), dataServerConfig .getLocalDataServerCleanDelay())); } } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/DataDigestResource.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/DataDigestResource.java index 786bf7af0..d945dc525 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/DataDigestResource.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/DataDigestResource.java @@ -16,21 +16,37 @@ */ package com.alipay.sofa.registry.server.data.resource; -import com.alipay.sofa.registry.common.model.dataserver.Datum; -import com.alipay.sofa.registry.common.model.store.DataInfo; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.net.NetUtil; -import com.alipay.sofa.registry.server.data.cache.DatumCache; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.remoting.Connection; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.DataServerCache; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.node.DataServerNode; +import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerNodeFactory; +import com.alipay.sofa.registry.server.data.remoting.metaserver.MetaServerConnectionFactory; +import com.alipay.sofa.registry.server.data.remoting.sessionserver.SessionServerConnectionFactory; /** * @@ -40,6 +56,27 @@ @Path("digest") public class DataDigestResource { + private final static String SESSION = "SESSION"; + + private final static String DATA = "DATA"; + + private final static String META = "META"; + + @Autowired + private SessionServerConnectionFactory sessionServerConnectionFactory; + + @Autowired + private MetaServerConnectionFactory metaServerConnectionFactory; + + @Autowired + private DataServerConfig dataServerConfig; + + @Autowired + private DatumCache datumCache; + + @Autowired + private DataServerCache dataServerCache; + @GET @Path("datum/query") @Produces(MediaType.APPLICATION_JSON) @@ -53,9 +90,9 @@ public Map getDatumByDataInfoId(@QueryParam("dataId") String data String dataInfoId = DataInfo.toDataInfoId(dataId, instanceId, group); if (isBlank(dataCenter)) { - retList = DatumCache.get(dataInfoId); + retList = datumCache.get(dataInfoId); } else { - retList.put(dataCenter, DatumCache.get(dataCenter, dataInfoId)); + retList.put(dataCenter, datumCache.get(dataCenter, dataInfoId)); } } @@ -68,10 +105,10 @@ public Map getDatumByDataInfoId(@QueryParam("dataId") String data public Map> getPublishersByConnectId(Map map) { Map> ret = new HashMap<>(); if (map != null && !map.isEmpty()) { - map.forEach((ip, port) -> { - String connectId = NetUtil.genHost(ip, Integer.valueOf(port)); + map.forEach((clientConnectId, sessionConnectId) -> { + String connectId = clientConnectId + ValueConstants.CONNECT_ID_SPLIT + sessionConnectId; if (!connectId.isEmpty()) { - Map publisherMap = DatumCache.getByHost(connectId); + Map publisherMap = datumCache.getByConnectId(connectId); if (publisherMap != null && !publisherMap.isEmpty()) { ret.put(connectId, publisherMap); } @@ -85,20 +122,22 @@ public Map> getPublishersByConnectId(Map> allMap = DatumCache.getAll(); + Map> allMap = datumCache.getAll(); if (!allMap.isEmpty()) { for (Entry> dataCenterEntry : allMap.entrySet()) { String dataCenter = dataCenterEntry.getKey(); Map datumMap = dataCenterEntry.getValue(); - sb.append(String.format(" [Datum] size of datum in %s is %s", - dataCenter, datumMap.size())); + sb.append(String.format(" [Datum] size of datum in %s is %s", dataCenter, datumMap.size())); int pubCount = datumMap.values().stream().map(Datum::getPubMap) .filter(map -> map != null && !map.isEmpty()).mapToInt(Map::size).sum(); - sb.append(String.format(",[Publisher] size of publisher in %s is %s", - dataCenter, pubCount)); + sb.append(String.format(",[Publisher] size of publisher in %s is %s", dataCenter, pubCount)); } } else { sb.append(" datum cache is empty"); @@ -111,6 +150,91 @@ public String getDatumCount() { return sb.toString(); } + @GET + @Path("{type}/serverList/query") + @Produces(MediaType.APPLICATION_JSON) + public Map> getServerListAll(@PathParam("type") String type) { + + Map> map = new HashMap<>(); + if (type != null && !type.isEmpty()) { + String inputType = type.toUpperCase(); + + switch (inputType) { + case SESSION: + List sessionList = getSessionServerList(); + if (sessionList != null) { + map = new HashMap<>(); + map.put(dataServerConfig.getLocalDataCenter(), sessionList); + } + break; + case DATA: + map = getDataServerList(); + break; + case META: + map = getMetaServerList(); + break; + default: + map = new HashMap<>(); + break; + } + + } + return map; + } + + public List getSessionServerList() { + List connections = sessionServerConnectionFactory.getSessionConnections().stream() + .filter(connection -> connection != null && connection.isFine()) + .map(connection -> connection.getRemoteIP() + ":" + connection.getRemotePort()) + .collect(Collectors.toList()); + return connections; + } + + public Map> getDataServerList() { + + Map> map = new HashMap<>(); + Set allDataCenter = new HashSet<>(dataServerCache.getAllDataCenters()); + for (String dataCenter : allDataCenter) { + + List list = map.computeIfAbsent(dataCenter, k -> new ArrayList<>()); + + Map dataNodes = DataServerNodeFactory.getDataServerNodes(dataCenter); + if (dataNodes != null && !dataNodes.isEmpty()) { + + dataNodes.forEach((ip, dataServerNode) -> { + if (ip != null && !ip.equals(DataServerConfig.IP)) { + Connection connection = dataServerNode.getConnection(); + if (connection != null && connection.isFine()) { + list.add(connection.getRemoteIP()); + } + } + }); + } + } + return map; + } + + public Map> getMetaServerList() { + + Map> map = new HashMap<>(); + Set allDataCenter = new HashSet<>(metaServerConnectionFactory.getAllDataCenters()); + for (String dataCenter : allDataCenter) { + + List list = map.computeIfAbsent(dataCenter, k -> new ArrayList<>()); + + Map metaConnections = metaServerConnectionFactory.getConnections(dataCenter); + if (metaConnections != null && !metaConnections.isEmpty()) { + + metaConnections.forEach((ip, connection) -> { + if (connection != null && connection.isFine()) { + list.add(connection.getRemoteIP()); + } + }); + } + } + return map; + } + private boolean isBlank(String dataInfoId) { return dataInfoId == null || dataInfoId.isEmpty(); } diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/HealthResource.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/HealthResource.java index c61a8bcf8..99e867c8a 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/HealthResource.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/resource/HealthResource.java @@ -16,16 +16,24 @@ */ package com.alipay.sofa.registry.server.data.resource; -import com.alipay.sofa.registry.common.model.CommonResponse; -import com.alipay.sofa.registry.server.data.bootstrap.DataServerBootstrap; -import com.alipay.sofa.registry.server.data.node.DataNodeStatus; -import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; -import org.springframework.beans.factory.annotation.Autowired; - +import javax.annotation.PostConstruct; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.Response.Status; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.metrics.ReporterUtils; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerBootstrap; +import com.alipay.sofa.registry.server.data.node.DataNodeStatus; +import com.alipay.sofa.registry.server.data.util.LocalServerStatusEnum; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; /** * @@ -41,11 +49,29 @@ public class HealthResource { @Autowired private DataNodeStatus dataNodeStatus; + @PostConstruct + public void init() { + MetricRegistry metrics = new MetricRegistry(); + metrics.register("healthCheck", (Gauge) () -> getHealthCheckResult()); + ReporterUtils.startSlf4jReporter(60, metrics); + } + @GET @Path("check") @Produces(MediaType.APPLICATION_JSON) - public CommonResponse checkHealth() { + public Response checkHealth() { + + ResponseBuilder builder = Response.status(Response.Status.OK); + CommonResponse response = getHealthCheckResult(); + builder.entity(response); + if (!response.isSuccess()) { + builder.status(Status.INTERNAL_SERVER_ERROR); + } + + return builder.build(); + } + private CommonResponse getHealthCheckResult() { CommonResponse response; StringBuilder sb = new StringBuilder("DataServerBoot "); @@ -76,6 +102,11 @@ public CommonResponse checkHealth() { response = CommonResponse.buildFailedResponse(sb.toString()); } + if (ret) { + response = CommonResponse.buildSuccessResponse(sb.toString()); + } else { + response = CommonResponse.buildFailedResponse(sb.toString()); + } return response; } } \ No newline at end of file diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/ThreadPoolExecutorDataServer.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/DataMetricsThreadPoolExecutor.java similarity index 71% rename from server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/ThreadPoolExecutorDataServer.java rename to server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/DataMetricsThreadPoolExecutor.java index 427e15f4b..ce303c2c2 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/ThreadPoolExecutorDataServer.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/util/DataMetricsThreadPoolExecutor.java @@ -18,6 +18,7 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.metrics.TaskMetrics; import java.util.concurrent.BlockingQueue; import java.util.concurrent.RejectedExecutionException; @@ -30,19 +31,20 @@ * @author shangyu.wh * @version $Id: ThreadPoolExecutorDataServer.java, v 0.1 2018-10-25 20:40 shangyu.wh Exp $ */ -public class ThreadPoolExecutorDataServer extends ThreadPoolExecutor { +public class DataMetricsThreadPoolExecutor extends ThreadPoolExecutor { private static final Logger LOGGER = LoggerFactory - .getLogger(ThreadPoolExecutorDataServer.class); + .getLogger(DataMetricsThreadPoolExecutor.class); private String executorName; - public ThreadPoolExecutorDataServer(String executorName, int corePoolSize, int maximumPoolSize, - long keepAliveTime, TimeUnit unit, - BlockingQueue workQueue, - ThreadFactory threadFactory) { + public DataMetricsThreadPoolExecutor(String executorName, int corePoolSize, + int maximumPoolSize, long keepAliveTime, TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); - this.executorName = executorName; + this.executorName = "Data-" + executorName; + registerTaskMetrics(); } @Override @@ -50,6 +52,10 @@ public String toString() { return super.toString() + executorName; } + private void registerTaskMetrics() { + TaskMetrics.getInstance().registerThreadExecutor(executorName, this); + } + @Override public void execute(Runnable command) { try { diff --git a/server/server/data/src/main/resources/application.properties b/server/server/data/src/main/resources/application.properties index 77c9484d3..1cfb57b29 100644 --- a/server/server/data/src/main/resources/application.properties +++ b/server/server/data/src/main/resources/application.properties @@ -1,6 +1,6 @@ spring.main.banner-mode=LOG -#nodes.metaNode=: -#nodes.localDataCenter= +nodes.metaNode=DefaultDataCenter:localhost +nodes.localDataCenter=DefaultDataCenter #data.server.logging.level=INFO #data.server.logging.home=/home/admin/logs/registry/data @@ -10,8 +10,13 @@ data.server.httpServerPort=9622 data.server.queueCount=4 data.server.queueSize=10240 data.server.notifyIntervalMs=500 +data.server.clientOffDelayMs=1000 +data.server.notifyTempDataIntervalMs=10 data.server.rpcTimeout=3000 data.server.metaServerPort=9611 data.server.storeNodes=3 data.server.numberOfReplicas=1000 +data.server.datumTimeToLiveSec=900 +data.server.sessionDisconnectDelayMs=30000 +data.server.metricsExecutor.fixedDelay=30000 diff --git a/server/server/data/src/main/resources/logback-spring.xml b/server/server/data/src/main/resources/logback-spring.xml index eab8360d6..f7dfec237 100644 --- a/server/server/data/src/main/resources/logback-spring.xml +++ b/server/server/data/src/main/resources/logback-spring.xml @@ -30,6 +30,25 @@ ${LOG_ENCODE} + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/profile-digest.log + + ${DATA_LOG_HOME}/profile-digest.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + true @@ -71,6 +90,26 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-renew.log + + ${DATA_LOG_HOME}/registry-renew.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -85,7 +124,7 @@ ${DATA_LOG_HOME}/registry-data.log ${DATA_LOG_HOME}/registry-data.log.%d{yyyy-MM-dd} - 30 + 3 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -144,8 +183,8 @@ ${DATA_LOG_HOME}/cache-digest.log - ${DATA_LOG_HOME}/cache-digest.log.%d{yyyy-MM-dd} - 30 + ${DATA_LOG_HOME}/cache-digest.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -173,6 +212,86 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-connect.log + + ${DATA_LOG_HOME}/registry-connect.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-exchange.log + + ${DATA_LOG_HOME}/registry-exchange.log.%d{yyyy-MM-dd_HH} + 72 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-metrics.log + + ${DATA_LOG_HOME}/registry-metrics.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-datum-lease.log + + ${DATA_LOG_HOME}/registry-datum-lease.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -185,6 +304,11 @@ + + + + + @@ -198,7 +322,7 @@ - @@ -214,6 +338,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/server/data/src/main/resources/security/blacklist.txt b/server/server/data/src/main/resources/security/blacklist.txt new file mode 100644 index 000000000..e3413f8d4 --- /dev/null +++ b/server/server/data/src/main/resources/security/blacklist.txt @@ -0,0 +1,78 @@ +org.codehaus.groovy.runtime.MethodClosure +clojure.core$constantly +clojure.main$eval_opt +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactory +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactoryImpl +com.alibaba.citrus.springext.util.SpringExtUtil.AbstractProxy +com.alipay.custrelation.service.model.redress.Pair +com.caucho.hessian.test.TestCons +com.mchange.v2.c3p0.JndiRefForwardingDataSource +com.mchange.v2.c3p0.WrapperConnectionPoolDataSource +com.rometools.rome.feed.impl.EqualsBean +com.rometools.rome.feed.impl.ToStringBean +com.sun.jndi.rmi.registry.BindingEnumeration +com.sun.jndi.toolkit.dir.LazySearchEnumerationImpl +com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl +com.sun.rowset.JdbcRowSetImpl +com.sun.xml.internal.bind.v2.runtime.unmarshaller.Base64Data +java.rmi.server.UnicastRemoteObject +java.security.SignedObject +java.util.ServiceLoader$LazyIterator +javax.imageio.ImageIO$ContainsFilter +javax.imageio.spi.ServiceRegistry +javax.management.BadAttributeValueExpException +javax.naming.InitialContext +javax.naming.spi.ObjectFactory +javax.script.ScriptEngineManager +javax.sound.sampled.AudioFormat$Encoding +org.apache.carbondata.core.scan.expression.ExpressionResult +org.apache.commons.dbcp +org.apache.commons.dbcp2 +org.apache.commons.beanutils +org.apache.ibatis.executor.loader.AbstractSerialStateHolder +org.apache.ibatis.executor.loader.CglibSerialStateHolder +org.apache.ibatis.executor.loader.JavassistSerialStateHolder +org.apache.ibatis.executor.loader.cglib.CglibProxyFactory +org.apache.ibatis.executor.loader.javassist.JavassistSerialStateHolder +org.apache.tomcat.dbcp.dbcp.datasources.SharedPoolDataSource +org.apache.wicket.util.upload.DiskFileItem +org.apache.xalan.xsltc.trax.TemplatesImpl +org.apache.xbean.naming.context.ContextUtil$ReadOnlyBinding +org.apache.xpath.XPathContext +org.eclipse.jetty.util.log.LoggerLog +org.geotools.filter.ConstantExpression +org.springframework.aop.aspectj.autoproxy.AspectJAwareAdvisorAutoProxyCreator$PartiallyComparableAdvisorHolder +org.springframework.aop.support.DefaultBeanFactoryPointcutAdvisor +org.springframework.beans.factory.BeanFactory +org.springframework.beans.factory.config.PropertyPathFactoryBean +org.springframework.beans.factory.support.DefaultListableBeanFactory +org.springframework.jndi.support.SimpleJndiBeanFactory +org.springframework.orm.jpa.AbstractEntityManagerFactoryBean +org.springframework.transaction.jta.JtaTransactionManager +org.yaml.snakeyaml.tokens.DirectiveToken +sun.rmi.server.UnicastRef +javax.management.ImmutableDescriptor +org.springframework.jndi.JndiObjectTargetSource +ch.qos.logback.core.db.JNDIConnectionSource +java.beans.Expression +javassist.bytecode +javassist.tools.web.Viewer +javassist.util.proxy.SerializedProxy +org.apache.ibatis.javassist.bytecode +org.apache.ibatis.javassist.tools.web.Viewer +org.apache.ibatis.javassist.util.proxy.SerializedProxy +org.springframework.beans.factory.config.MethodInvokingFactoryBean +com.alibaba.druid +com.sun.corba +com.sun.org.apache.bcel +com.sun.org.apache.xml +com.sun.org.apache.xpath +org.apache.zookeeper.Shell +org.apache.tomcat.dbcp.dbcp.BasicDataSource +net.bytebuddy.dynamic.loading.ByteArrayClassLoader +org.jboss.resteasy.plugins +org.springframework.beans.BeanWrapperImpl$BeanPropertyHandler +org.apache.velocity.runtime.resource.Resource +org.springframework.expression.spel.ast.Indexer$PropertyIndexingValueRef +org.mortbay.log.Slf4jLog +org.springframework.expression.spel.ast.MethodReference$MethodValueRef \ No newline at end of file diff --git a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/BackupTriadTest.java b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/BackupTriadTest.java index ba4bba93a..f601b07f0 100644 --- a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/BackupTriadTest.java +++ b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/BackupTriadTest.java @@ -16,20 +16,21 @@ */ package com.alipay.sofa.registry.server.data; -import com.alipay.sofa.registry.common.model.metaserver.DataNode; -import com.alipay.sofa.registry.common.model.store.URL; -import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; -import com.alipay.sofa.registry.server.data.cache.BackupTriad; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import org.junit.Test; + +import com.alipay.sofa.registry.common.model.metaserver.DataNode; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.server.data.bootstrap.DataServerConfig; +import com.alipay.sofa.registry.server.data.cache.BackupTriad; /** * @author xuanbei @@ -51,12 +52,10 @@ public void doTest() { Set notWorking = new HashSet<>(); notWorking.add("192.168.0.2"); assertEquals(2, backupTriad.getNewJoined(newTriad, notWorking).size()); - assertEquals( - "DataNode{ip=192.168.0.2, dataCenter='DefaultDataCenter', regionId='null', nodeStatus=INIT, registrationTimestamp=0}", - backupTriad.getNewJoined(newTriad, notWorking).get(0).toString()); - assertEquals( - "DataNode{ip=192.168.0.4, dataCenter='DefaultDataCenter', regionId='null', nodeStatus=INIT, registrationTimestamp=0}", - backupTriad.getNewJoined(newTriad, notWorking).get(1).toString()); + assertEquals("DataNode{ip=192.168.0.2}", backupTriad.getNewJoined(newTriad, notWorking) + .get(0).toString()); + assertEquals("DataNode{ip=192.168.0.4}", backupTriad.getNewJoined(newTriad, notWorking) + .get(1).toString()); assertEquals("TestDataInfoId", backupTriad.getDataInfoId()); backupTriad.setDataInfoId("AnotherTestDataInfoId"); @@ -68,14 +67,9 @@ public void doTest() { backupTriad.setTriad(nodeList); assertTrue(backupTriad.containsSelf()); assertEquals(2, backupTriad.getTriad().size()); - assertEquals( - "DataNode{ip=192.168.0.1, dataCenter='DefaultDataCenter', regionId='null', nodeStatus=INIT, registrationTimestamp=0}", - backupTriad.getTriad().get(0).toString()); - assertEquals( - "DataNode{ip=" - + DataServerConfig.IP - + ", dataCenter='DefaultDataCenter', regionId='null', nodeStatus=INIT, registrationTimestamp=0}", - backupTriad.getTriad().get(1).toString()); + assertEquals("DataNode{ip=192.168.0.1}", backupTriad.getTriad().get(0).toString()); + assertEquals("DataNode{ip=" + DataServerConfig.IP + "}", backupTriad.getTriad().get(1) + .toString()); assertTrue(backupTriad.toString().contains( "BackupTriad{dataInfoId='AnotherTestDataInfoId', ipSetOfNode=") && backupTriad.toString().contains("192.168.0.1") diff --git a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/TestSyncData.java b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/TestSyncData.java index d1cf8ebad..26044eeed 100644 --- a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/TestSyncData.java +++ b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/TestSyncData.java @@ -16,15 +16,18 @@ */ package com.alipay.sofa.registry.server.data; -import com.alipay.sofa.registry.common.model.dataserver.Datum; -import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; -import com.alipay.sofa.registry.server.data.datasync.Operator; -import com.alipay.sofa.registry.server.data.datasync.sync.Acceptor; +import java.util.Collection; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.Collection; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.server.data.change.DataSourceTypeEnum; +import com.alipay.sofa.registry.server.data.datasync.Operator; +import com.alipay.sofa.registry.server.data.datasync.sync.Acceptor; +import com.alipay.sofa.registry.util.DatumVersionUtil; /** * @@ -75,15 +78,15 @@ public void setup() { @Test public void testAcceptExpired() throws InterruptedException { - Acceptor acceptor = new Acceptor(30, "11", "DefaultDataCenter"); + Acceptor acceptor = new Acceptor(30, "11", "DefaultDataCenter", new DatumCache()); - Operator operator1 = new Operator(System.currentTimeMillis(), 0L, datum1, + Operator operator1 = new Operator(DatumVersionUtil.nextId(), 0L, datum1, DataSourceTypeEnum.SYNC); Thread.sleep(1000); - Operator operator2 = new Operator(System.currentTimeMillis(), operator1.getVersion(), + Operator operator2 = new Operator(DatumVersionUtil.nextId(), operator1.getVersion(), datum2, DataSourceTypeEnum.SYNC); Thread.sleep(2000); - Operator operator5 = new Operator(System.currentTimeMillis(), operator2.getVersion(), + Operator operator5 = new Operator(DatumVersionUtil.nextId(), operator2.getVersion(), datum5, DataSourceTypeEnum.SYNC); acceptor.appendOperator(operator1); @@ -100,4 +103,5 @@ public void testAcceptExpired() throws InterruptedException { Assert.assertTrue(ops[0].getVersion().equals(operator2.getVersion())); Assert.assertTrue(ops[1].getVersion().equals(operator5.getVersion())); } + } \ No newline at end of file diff --git a/server/server/data/src/test/resources/logback-test.xml b/server/server/data/src/test/resources/logback-test.xml index 349319ca3..9ff8d8bf5 100644 --- a/server/server/data/src/test/resources/logback-test.xml +++ b/server/server/data/src/test/resources/logback-test.xml @@ -39,7 +39,7 @@ ${LOG_HOME}/registry-data.log ${LOG_HOME}/registry-data.log.%d{yyyy-MM-dd} - 30 + 3 [%d{ISO8601}][%p][%t][%c{0}] - %m%n diff --git a/server/server/integration/pom.xml b/server/server/integration/pom.xml index d5cd0191d..5540b81fd 100644 --- a/server/server/integration/pom.xml +++ b/server/server/integration/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/server/integration/src/main/resources/application.properties b/server/server/integration/src/main/resources/application.properties index 08a6b78f2..62ee2587a 100644 --- a/server/server/integration/src/main/resources/application.properties +++ b/server/server/integration/src/main/resources/application.properties @@ -1,7 +1,7 @@ spring.main.banner-mode=LOG -#nodes.metaNode=: -#nodes.localDataCenter= -#nodes.localRegion= +nodes.metaNode=DefaultDataCenter:localhost +nodes.localDataCenter=DefaultDataCenter +nodes.localRegion=DEFAULT_ZONE #server.logging.home=/home/admin/logs/registry #meta.server.logging.level=INFO @@ -12,6 +12,7 @@ meta.server.metaServerPort=9612 meta.server.raftServerPort=9614 meta.server.httpServerPort=9615 meta.server.raftGroup=MetaServerRaftGroup +meta.server.metricsExecutor.fixedDelay=30000 #data.server.logging.level=INFO #data.server.logging.home=/home/admin/logs/registry/data @@ -25,6 +26,7 @@ data.server.rpcTimeout=3000 data.server.metaServerPort=9611 data.server.storeNodes=3 data.server.numberOfReplicas=1000 +data.server.metricsExecutor.fixedDelay=30000 #session.server.logging.level=INFO #session.server.logging.home=/home/admin/logs/registry/session @@ -42,9 +44,9 @@ session.server.printTask.fixedDelay=30000 session.server.schedulerCheckVersionTimeout=3 session.server.schedulerCheckVersionFirstDelay=3 session.server.schedulerCheckVersionExpBackOffBound=10 -session.server.schedulerHeartbeatTimeout=30 -session.server.schedulerHeartbeatFirstDelay=30 -session.server.schedulerHeartbeatExpBackOffBound=10 +session.server.schedulerHeartbeatTimeout=3 +session.server.schedulerHeartbeatFirstDelay=3 +session.server.schedulerHeartbeatExpBackOffBound=1 session.server.schedulerFetchDataTimeout=3 session.server.schedulerFetchDataFirstDelay=3 session.server.schedulerFetchDataExpBackOffBound=10 diff --git a/server/server/integration/src/main/resources/logback-spring.xml b/server/server/integration/src/main/resources/logback-spring.xml index 167ae8744..44e0496be 100644 --- a/server/server/integration/src/main/resources/logback-spring.xml +++ b/server/server/integration/src/main/resources/logback-spring.xml @@ -97,7 +97,46 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${LOG_HOME}/registry-renew.log + + ${LOG_HOME}/registry-renew.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${META_LOG_HOME}/profile-digest.log + + ${META_LOG_HOME}/profile-digest.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + true @@ -193,7 +232,7 @@ ${DATA_LOG_HOME}/registry-data.log ${DATA_LOG_HOME}/registry-data.log.%d{yyyy-MM-dd} - 30 + 3 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -232,8 +271,8 @@ ${DATA_LOG_HOME}/cache-digest.log - ${DATA_LOG_HOME}/cache-digest.log.%d{yyyy-MM-dd} - 30 + ${DATA_LOG_HOME}/cache-digest.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -281,6 +320,26 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${DATA_LOG_HOME}/registry-datum-lease.log + + ${DATA_LOG_HOME}/registry-datum-lease.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -295,7 +354,7 @@ ${SESSION_LOG_HOME}/registry-session.log ${SESSION_LOG_HOME}/registry-session.log.%d{yyyy-MM-dd} - 30 + 3 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -354,8 +413,8 @@ ${SESSION_LOG_HOME}/registry-exchange.log - ${SESSION_LOG_HOME}/registry-exchange.log.%d{yyyy-MM-dd} - 30 + ${SESSION_LOG_HOME}/registry-exchange.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -374,8 +433,8 @@ ${SESSION_LOG_HOME}/registry-push.log - ${SESSION_LOG_HOME}/registry-push.log.%d{yyyy-MM-dd} - 30 + ${SESSION_LOG_HOME}/registry-push.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -451,6 +510,11 @@ + + + + + @@ -496,6 +560,11 @@ + + + + + @@ -527,6 +596,15 @@ + + + + + + + + + diff --git a/server/server/integration/src/main/resources/security/blacklist.txt b/server/server/integration/src/main/resources/security/blacklist.txt new file mode 100644 index 000000000..e3413f8d4 --- /dev/null +++ b/server/server/integration/src/main/resources/security/blacklist.txt @@ -0,0 +1,78 @@ +org.codehaus.groovy.runtime.MethodClosure +clojure.core$constantly +clojure.main$eval_opt +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactory +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactoryImpl +com.alibaba.citrus.springext.util.SpringExtUtil.AbstractProxy +com.alipay.custrelation.service.model.redress.Pair +com.caucho.hessian.test.TestCons +com.mchange.v2.c3p0.JndiRefForwardingDataSource +com.mchange.v2.c3p0.WrapperConnectionPoolDataSource +com.rometools.rome.feed.impl.EqualsBean +com.rometools.rome.feed.impl.ToStringBean +com.sun.jndi.rmi.registry.BindingEnumeration +com.sun.jndi.toolkit.dir.LazySearchEnumerationImpl +com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl +com.sun.rowset.JdbcRowSetImpl +com.sun.xml.internal.bind.v2.runtime.unmarshaller.Base64Data +java.rmi.server.UnicastRemoteObject +java.security.SignedObject +java.util.ServiceLoader$LazyIterator +javax.imageio.ImageIO$ContainsFilter +javax.imageio.spi.ServiceRegistry +javax.management.BadAttributeValueExpException +javax.naming.InitialContext +javax.naming.spi.ObjectFactory +javax.script.ScriptEngineManager +javax.sound.sampled.AudioFormat$Encoding +org.apache.carbondata.core.scan.expression.ExpressionResult +org.apache.commons.dbcp +org.apache.commons.dbcp2 +org.apache.commons.beanutils +org.apache.ibatis.executor.loader.AbstractSerialStateHolder +org.apache.ibatis.executor.loader.CglibSerialStateHolder +org.apache.ibatis.executor.loader.JavassistSerialStateHolder +org.apache.ibatis.executor.loader.cglib.CglibProxyFactory +org.apache.ibatis.executor.loader.javassist.JavassistSerialStateHolder +org.apache.tomcat.dbcp.dbcp.datasources.SharedPoolDataSource +org.apache.wicket.util.upload.DiskFileItem +org.apache.xalan.xsltc.trax.TemplatesImpl +org.apache.xbean.naming.context.ContextUtil$ReadOnlyBinding +org.apache.xpath.XPathContext +org.eclipse.jetty.util.log.LoggerLog +org.geotools.filter.ConstantExpression +org.springframework.aop.aspectj.autoproxy.AspectJAwareAdvisorAutoProxyCreator$PartiallyComparableAdvisorHolder +org.springframework.aop.support.DefaultBeanFactoryPointcutAdvisor +org.springframework.beans.factory.BeanFactory +org.springframework.beans.factory.config.PropertyPathFactoryBean +org.springframework.beans.factory.support.DefaultListableBeanFactory +org.springframework.jndi.support.SimpleJndiBeanFactory +org.springframework.orm.jpa.AbstractEntityManagerFactoryBean +org.springframework.transaction.jta.JtaTransactionManager +org.yaml.snakeyaml.tokens.DirectiveToken +sun.rmi.server.UnicastRef +javax.management.ImmutableDescriptor +org.springframework.jndi.JndiObjectTargetSource +ch.qos.logback.core.db.JNDIConnectionSource +java.beans.Expression +javassist.bytecode +javassist.tools.web.Viewer +javassist.util.proxy.SerializedProxy +org.apache.ibatis.javassist.bytecode +org.apache.ibatis.javassist.tools.web.Viewer +org.apache.ibatis.javassist.util.proxy.SerializedProxy +org.springframework.beans.factory.config.MethodInvokingFactoryBean +com.alibaba.druid +com.sun.corba +com.sun.org.apache.bcel +com.sun.org.apache.xml +com.sun.org.apache.xpath +org.apache.zookeeper.Shell +org.apache.tomcat.dbcp.dbcp.BasicDataSource +net.bytebuddy.dynamic.loading.ByteArrayClassLoader +org.jboss.resteasy.plugins +org.springframework.beans.BeanWrapperImpl$BeanPropertyHandler +org.apache.velocity.runtime.resource.Resource +org.springframework.expression.spel.ast.Indexer$PropertyIndexingValueRef +org.mortbay.log.Slf4jLog +org.springframework.expression.spel.ast.MethodReference$MethodValueRef \ No newline at end of file diff --git a/server/server/meta/pom.xml b/server/server/meta/pom.xml index 9c8ded95a..3460620f5 100644 --- a/server/server/meta/pom.xml +++ b/server/server/meta/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/MetaApplication.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/MetaApplication.java index 0869d1851..1f98d4987 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/MetaApplication.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/MetaApplication.java @@ -21,6 +21,7 @@ import com.alipay.sofa.registry.server.meta.bootstrap.EnableMetaServer; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; /** * @@ -28,6 +29,7 @@ * @version $Id: MetaApplication.java, v 0.1 2017-11-13 19:03 zhuoyu.sjw Exp $$ */ @EnableMetaServer +@EnableScheduling @SpringBootApplication public class MetaApplication { @@ -36,7 +38,9 @@ public class MetaApplication { public static void main(String[] args) { // setup DefaultUncaughtExceptionHandler Thread.setDefaultUncaughtExceptionHandler((t, e) -> { - LOGGER.error(String.format("UncaughtException in Thread(%s): %s", t.getName(), e.getMessage()), e); + LOGGER.error( + String.format("UncaughtException in Thread(%s): %s", t.getName(), e.getMessage()), + e); }); SpringApplication.run(MetaApplication.class, args); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/AbstractNodeConfigBean.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/AbstractNodeConfigBean.java index c5b8e78a5..ddcb5e4b5 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/AbstractNodeConfigBean.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/AbstractNodeConfigBean.java @@ -16,10 +16,6 @@ */ package com.alipay.sofa.registry.server.meta.bootstrap; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.net.NetUtil; - import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -29,6 +25,10 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.net.NetUtil; + /** * * @author shangyu.wh @@ -105,9 +105,6 @@ public String getMetaDataCenter(String metaIpAddress) { } }); - if (ret.get() == null) { - LOGGER.error("node ipAddress:" + metaIpAddress + " cannot be found on config list!"); - } dataCenterRet = ret.get(); } return dataCenterRet; diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerBootstrap.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerBootstrap.java index 613b338cf..9e700c199 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerBootstrap.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerBootstrap.java @@ -16,6 +16,20 @@ */ package com.alipay.sofa.registry.server.meta.bootstrap; +import java.lang.annotation.Annotation; +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Resource; +import javax.ws.rs.Path; +import javax.ws.rs.ext.Provider; + +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -27,18 +41,6 @@ import com.alipay.sofa.registry.server.meta.remoting.RaftExchanger; import com.alipay.sofa.registry.server.meta.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.task.batcher.TaskDispatchers; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; - -import javax.annotation.Resource; -import javax.ws.rs.Path; -import javax.ws.rs.ext.Provider; -import java.lang.annotation.Annotation; -import java.util.Collection; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** * @@ -99,8 +101,10 @@ public class MetaServerBootstrap { /** * Do initialized. */ - public void doInitialized() { + public void start() { try { + LOGGER.info("the configuration items are as follows: " + metaServerConfig.toString()); + openSessionRegisterServer(); openDataRegisterServer(); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfig.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfig.java index d4b09b38b..0046481b0 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfig.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfig.java @@ -17,7 +17,6 @@ package com.alipay.sofa.registry.server.meta.bootstrap; /** - * * @author shangyu.wh * @version $Id: MetaServerConfig.java, v 0.1 2018-01-16 10:58 shangyu.wh Exp $ */ @@ -80,6 +79,74 @@ public interface MetaServerConfig { boolean isEnableMetrics(); + int getRockDBCacheSize(); + + int getHeartbeatCheckExecutorMinSize(); + + int getHeartbeatCheckExecutorMaxSize(); + + int getHeartbeatCheckExecutorQueueSize(); + + int getCheckDataChangeExecutorMinSize(); + + int getCheckDataChangeExecutorMaxSize(); + + int getCheckDataChangeExecutorQueueSize(); + + int getGetOtherDataCenterChangeExecutorMinSize(); + + int getGetOtherDataCenterChangeExecutorMaxSize(); + + int getGetOtherDataCenterChangeExecutorQueueSize(); + + int getConnectMetaServerExecutorMinSize(); + + int getConnectMetaServerExecutorMaxSize(); + + int getConnectMetaServerExecutorQueueSize(); + + int getCheckNodeListChangePushExecutorMinSize(); + + int getCheckNodeListChangePushExecutorMaxSize(); + + int getCheckNodeListChangePushExecutorQueueSize(); + + int getRaftClientRefreshExecutorMinSize(); + + int getRaftClientRefreshExecutorMaxSize(); + + int getRaftClientRefreshExecutorQueueSize(); + + int getMetaSchedulerPoolSize(); + + int getDefaultRequestExecutorMinSize(); + + int getDefaultRequestExecutorMaxSize(); + + int getDefaultRequestExecutorQueueSize(); + + int getRaftExecutorMinSize(); + + int getRaftExecutorQueueSize(); + + int getRaftExecutorMaxSize(); + + int getRaftServerExecutorMinSize(); + + int getRaftServerExecutorMaxSize(); + + int getRaftServerExecutorQueueSize(); + + int getRaftFsmExecutorMinSize(); + + int getRaftFsmExecutorMaxSize(); + + int getRaftFsmExecutorQueueSize(); + + int getRaftElectionTimeout(); + + double getSessionLoadbalanceThresholdRatio(); + /** * decision mode enum */ diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfigBean.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfigBean.java index bb484c923..c9d843db7 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfigBean.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfigBean.java @@ -16,13 +16,15 @@ */ package com.alipay.sofa.registry.server.meta.bootstrap; -import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import java.io.File; + +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; import org.springframework.boot.context.properties.ConfigurationProperties; -import java.io.File; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; /** - * * @author shangyu.wh * @version $Id: MetaServerConfigBean.java, v 0.1 2018-01-16 11:01 shangyu.wh Exp $ */ @@ -41,15 +43,15 @@ public class MetaServerConfigBean implements MetaServerConfig { private int schedulerHeartbeatTimeout = 3; - private int schedulerHeartbeatFirstDelay = 3; + private int schedulerHeartbeatFirstDelay = 30; private int schedulerHeartbeatExpBackOffBound = 10; - private int schedulerGetDataChangeTimeout = 10; + private int schedulerGetDataChangeTimeout = 5; - private int schedulerGetDataChangeFirstDelay = 10; + private int schedulerGetDataChangeFirstDelay = 5; - private int schedulerGetDataChangeExpBackOffBound = 20; + private int schedulerGetDataChangeExpBackOffBound = 5; private int schedulerConnectMetaServerTimeout = 3; @@ -79,6 +81,8 @@ public class MetaServerConfigBean implements MetaServerConfig { private int sessionNodeChangePushTaskRetryTimes = 3; + private int raftElectionTimeout = 1000; + /** * Whether to enable metrics for node. */ @@ -91,6 +95,51 @@ public class MetaServerConfigBean implements MetaServerConfig { + File.separator + "raftData"; + private int rockDBCacheSize = 64; //64M + + private int heartbeatCheckExecutorMinSize = 3; + private int heartbeatCheckExecutorMaxSize = 10; + private int heartbeatCheckExecutorQueueSize = 1024; + + private int checkDataChangeExecutorMinSize = 3; + private int checkDataChangeExecutorMaxSize = 10; + private int checkDataChangeExecutorQueueSize = 1024; + + private int getOtherDataCenterChangeExecutorMinSize = 3; + private int getOtherDataCenterChangeExecutorMaxSize = 10; + private int getOtherDataCenterChangeExecutorQueueSize = 1024; + + private int connectMetaServerExecutorMinSize = 3; + private int connectMetaServerExecutorMaxSize = 10; + private int connectMetaServerExecutorQueueSize = 1024; + + private int checkNodeListChangePushExecutorMinSize = 3; + private int checkNodeListChangePushExecutorMaxSize = 10; + private int checkNodeListChangePushExecutorQueueSize = 1024; + + private int raftClientRefreshExecutorMinSize = 3; + private int raftClientRefreshExecutorMaxSize = 10; + private int raftClientRefreshExecutorQueueSize = 1024; + + private int defaultRequestExecutorMinSize = 20; + private int defaultRequestExecutorMaxSize = 600; + private int defaultRequestExecutorQueueSize = 500; + + private int raftExecutorMinSize = 20; + private int raftExecutorMaxSize = 400; + private int raftExecutorQueueSize = 100; + + private int raftServerExecutorMinSize = 20; + private int raftServerExecutorMaxSize = 100; + private int raftServerExecutorQueueSize = 100; + + private int raftFsmExecutorMinSize = 3; + private int raftFsmExecutorMaxSize = 10; + private int raftFsmExecutorQueueSize = 100; + + private int metaSchedulerPoolSize = 6; + private double sessionLoadbalanceThresholdRatio = 1.1; + @Override public int getSessionServerPort() { return sessionServerPort; @@ -99,7 +148,7 @@ public int getSessionServerPort() { /** * Setter method for property sessionServerPort. * - * @param sessionServerPort value to be assigned to property sessionServerPort + * @param sessionServerPort value to be assigned to property sessionServerPort */ public void setSessionServerPort(int sessionServerPort) { this.sessionServerPort = sessionServerPort; @@ -113,7 +162,7 @@ public int getDataServerPort() { /** * Setter method for property dataServerPort. * - * @param dataServerPort value to be assigned to property dataServerPort + * @param dataServerPort value to be assigned to property dataServerPort */ public void setDataServerPort(int dataServerPort) { this.dataServerPort = dataServerPort; @@ -127,7 +176,7 @@ public int getHttpServerPort() { /** * Setter method for property httpServerPort. * - * @param httpServerPort value to be assigned to property httpServerPort + * @param httpServerPort value to be assigned to property httpServerPort */ public void setHttpServerPort(int httpServerPort) { this.httpServerPort = httpServerPort; @@ -146,7 +195,7 @@ public int getSchedulerHeartbeatTimeout() { /** * Setter method for property schedulerHeartbeatTimeout. * - * @param schedulerHeartbeatTimeout value to be assigned to property schedulerHeartbeatTimeout + * @param schedulerHeartbeatTimeout value to be assigned to property schedulerHeartbeatTimeout */ public void setSchedulerHeartbeatTimeout(int schedulerHeartbeatTimeout) { this.schedulerHeartbeatTimeout = schedulerHeartbeatTimeout; @@ -165,7 +214,7 @@ public int getSchedulerHeartbeatFirstDelay() { /** * Setter method for property schedulerHeartbeatFirstDelay. * - * @param schedulerHeartbeatFirstDelay value to be assigned to property schedulerHeartbeatFirstDelay + * @param schedulerHeartbeatFirstDelay value to be assigned to property schedulerHeartbeatFirstDelay */ public void setSchedulerHeartbeatFirstDelay(int schedulerHeartbeatFirstDelay) { this.schedulerHeartbeatFirstDelay = schedulerHeartbeatFirstDelay; @@ -184,7 +233,7 @@ public int getSchedulerHeartbeatExpBackOffBound() { /** * Setter method for property schedulerHeartbeatExpBackOffBound. * - * @param schedulerHeartbeatExpBackOffBound value to be assigned to property schedulerHeartbeatExpBackOffBound + * @param schedulerHeartbeatExpBackOffBound value to be assigned to property schedulerHeartbeatExpBackOffBound */ public void setSchedulerHeartbeatExpBackOffBound(int schedulerHeartbeatExpBackOffBound) { this.schedulerHeartbeatExpBackOffBound = schedulerHeartbeatExpBackOffBound; @@ -198,7 +247,7 @@ public DecisionMode getDecisionMode() { /** * Setter method for property decisionMode. * - * @param decisionMode value to be assigned to property decisionMode + * @param decisionMode value to be assigned to property decisionMode */ public void setDecisionMode(DecisionMode decisionMode) { this.decisionMode = decisionMode; @@ -217,7 +266,7 @@ public int getDataNodeExchangeTimeout() { /** * Setter method for property dataNodeExchangeTimeout. * - * @param dataNodeExchangeTimeout value to be assigned to property dataNodeExchangeTimeout + * @param dataNodeExchangeTimeout value to be assigned to property dataNodeExchangeTimeout */ public void setDataNodeExchangeTimeout(int dataNodeExchangeTimeout) { this.dataNodeExchangeTimeout = dataNodeExchangeTimeout; @@ -236,7 +285,7 @@ public int getSessionNodeExchangeTimeout() { /** * Setter method for property sessionNodeExchangeTimeout. * - * @param sessionNodeExchangeTimeout value to be assigned to property sessionNodeExchangeTimeout + * @param sessionNodeExchangeTimeout value to be assigned to property sessionNodeExchangeTimeout */ public void setSessionNodeExchangeTimeout(int sessionNodeExchangeTimeout) { this.sessionNodeExchangeTimeout = sessionNodeExchangeTimeout; @@ -255,7 +304,7 @@ public int getSchedulerConnectMetaServerTimeout() { /** * Setter method for property schedulerConnectMetaServerTimeout. * - * @param schedulerConnectMetaServerTimeout value to be assigned to property schedulerConnectMetaServerTimeout + * @param schedulerConnectMetaServerTimeout value to be assigned to property schedulerConnectMetaServerTimeout */ public void setSchedulerConnectMetaServerTimeout(int schedulerConnectMetaServerTimeout) { this.schedulerConnectMetaServerTimeout = schedulerConnectMetaServerTimeout; @@ -274,7 +323,7 @@ public int getSchedulerConnectMetaServerFirstDelay() { /** * Setter method for property schedulerConnectMetaServerFirstDelay. * - * @param schedulerConnectMetaServerFirstDelay value to be assigned to property schedulerConnectMetaServerFirstDelay + * @param schedulerConnectMetaServerFirstDelay value to be assigned to property schedulerConnectMetaServerFirstDelay */ public void setSchedulerConnectMetaServerFirstDelay(int schedulerConnectMetaServerFirstDelay) { this.schedulerConnectMetaServerFirstDelay = schedulerConnectMetaServerFirstDelay; @@ -293,7 +342,7 @@ public int getSchedulerConnectMetaServerExpBackOffBound() { /** * Setter method for property schedulerConnectMetaServerExpBackOffBound. * - * @param schedulerConnectMetaServerExpBackOffBound value to be assigned to property schedulerConnectMetaServerExpBackOffBound + * @param schedulerConnectMetaServerExpBackOffBound value to be assigned to property schedulerConnectMetaServerExpBackOffBound */ public void setSchedulerConnectMetaServerExpBackOffBound(int schedulerConnectMetaServerExpBackOffBound) { this.schedulerConnectMetaServerExpBackOffBound = schedulerConnectMetaServerExpBackOffBound; @@ -312,7 +361,7 @@ public int getMetaServerPort() { /** * Setter method for property metaServerPort. * - * @param metaServerPort value to be assigned to property metaServerPort + * @param metaServerPort value to be assigned to property metaServerPort */ public void setMetaServerPort(int metaServerPort) { this.metaServerPort = metaServerPort; @@ -331,7 +380,7 @@ public int getMetaNodeExchangeTimeout() { /** * Setter method for property metaNodeExchangeTimeout. * - * @param metaNodeExchangeTimeout value to be assigned to property metaNodeExchangeTimeout + * @param metaNodeExchangeTimeout value to be assigned to property metaNodeExchangeTimeout */ public void setMetaNodeExchangeTimeout(int metaNodeExchangeTimeout) { this.metaNodeExchangeTimeout = metaNodeExchangeTimeout; @@ -350,7 +399,7 @@ public int getDataCenterChangeNotifyTaskRetryTimes() { /** * Setter method for property dataCenterChangeNotifyTaskRetryTimes. * - * @param dataCenterChangeNotifyTaskRetryTimes value to be assigned to property dataCenterChangeNotifyTaskRetryTimes + * @param dataCenterChangeNotifyTaskRetryTimes value to be assigned to property dataCenterChangeNotifyTaskRetryTimes */ public void setDataCenterChangeNotifyTaskRetryTimes(int dataCenterChangeNotifyTaskRetryTimes) { this.dataCenterChangeNotifyTaskRetryTimes = dataCenterChangeNotifyTaskRetryTimes; @@ -369,7 +418,7 @@ public int getDataNodeChangePushTaskRetryTimes() { /** * Setter method for property dataNodeChangePushTaskRetryTimes. * - * @param dataNodeChangePushTaskRetryTimes value to be assigned to property dataNodeChangePushTaskRetryTimes + * @param dataNodeChangePushTaskRetryTimes value to be assigned to property dataNodeChangePushTaskRetryTimes */ public void setDataNodeChangePushTaskRetryTimes(int dataNodeChangePushTaskRetryTimes) { this.dataNodeChangePushTaskRetryTimes = dataNodeChangePushTaskRetryTimes; @@ -388,7 +437,7 @@ public int getGetDataCenterChangeListTaskRetryTimes() { /** * Setter method for property getDataCenterChangeListTaskRetryTimes. * - * @param getDataCenterChangeListTaskRetryTimes value to be assigned to property getDataCenterChangeListTaskRetryTimes + * @param getDataCenterChangeListTaskRetryTimes value to be assigned to property getDataCenterChangeListTaskRetryTimes */ public void setGetDataCenterChangeListTaskRetryTimes(int getDataCenterChangeListTaskRetryTimes) { this.getDataCenterChangeListTaskRetryTimes = getDataCenterChangeListTaskRetryTimes; @@ -407,7 +456,7 @@ public int getReceiveStatusConfirmNotifyTaskRetryTimes() { /** * Setter method for property receiveStatusConfirmNotifyTaskRetryTimes. * - * @param receiveStatusConfirmNotifyTaskRetryTimes value to be assigned to property receiveStatusConfirmNotifyTaskRetryTimes + * @param receiveStatusConfirmNotifyTaskRetryTimes value to be assigned to property receiveStatusConfirmNotifyTaskRetryTimes */ public void setReceiveStatusConfirmNotifyTaskRetryTimes(int receiveStatusConfirmNotifyTaskRetryTimes) { this.receiveStatusConfirmNotifyTaskRetryTimes = receiveStatusConfirmNotifyTaskRetryTimes; @@ -426,7 +475,7 @@ public int getSessionNodeChangePushTaskRetryTimes() { /** * Setter method for property sessionNodeChangePushTaskRetryTimes. * - * @param sessionNodeChangePushTaskRetryTimes value to be assigned to property sessionNodeChangePushTaskRetryTimes + * @param sessionNodeChangePushTaskRetryTimes value to be assigned to property sessionNodeChangePushTaskRetryTimes */ public void setSessionNodeChangePushTaskRetryTimes(int sessionNodeChangePushTaskRetryTimes) { this.sessionNodeChangePushTaskRetryTimes = sessionNodeChangePushTaskRetryTimes; @@ -445,7 +494,7 @@ public int getSchedulerCheckNodeListChangePushTimeout() { /** * Setter method for property schedulerCheckNodeListChangePushTimeout. * - * @param schedulerCheckNodeListChangePushTimeout value to be assigned to property schedulerCheckNodeListChangePushTimeout + * @param schedulerCheckNodeListChangePushTimeout value to be assigned to property schedulerCheckNodeListChangePushTimeout */ public void setSchedulerCheckNodeListChangePushTimeout(int schedulerCheckNodeListChangePushTimeout) { this.schedulerCheckNodeListChangePushTimeout = schedulerCheckNodeListChangePushTimeout; @@ -464,7 +513,7 @@ public int getSchedulerCheckNodeListChangePushFirstDelay() { /** * Setter method for property schedulerCheckNodeListChangePushFirstDelay. * - * @param schedulerCheckNodeListChangePushFirstDelay value to be assigned to property schedulerCheckNodeListChangePushFirstDelay + * @param schedulerCheckNodeListChangePushFirstDelay value to be assigned to property schedulerCheckNodeListChangePushFirstDelay */ public void setSchedulerCheckNodeListChangePushFirstDelay(int schedulerCheckNodeListChangePushFirstDelay) { this.schedulerCheckNodeListChangePushFirstDelay = schedulerCheckNodeListChangePushFirstDelay; @@ -483,8 +532,8 @@ public int getSchedulerCheckNodeListChangePushExpBackOffBound() { /** * Setter method for property schedulerCheckNodeListChangePushExpBackOffBound. * - * @param schedulerCheckNodeListChangePushExpBackOffBound value to be assigned to property - * schedulerCheckNodeListChangePushExpBackOffBound + * @param schedulerCheckNodeListChangePushExpBackOffBound value to be assigned to property + * schedulerCheckNodeListChangePushExpBackOffBound */ public void setSchedulerCheckNodeListChangePushExpBackOffBound(int schedulerCheckNodeListChangePushExpBackOffBound) { this.schedulerCheckNodeListChangePushExpBackOffBound = schedulerCheckNodeListChangePushExpBackOffBound; @@ -503,7 +552,7 @@ public int getSchedulerGetDataChangeTimeout() { /** * Setter method for property schedulerGetDataChangeTimeout. * - * @param schedulerGetDataChangeTimeout value to be assigned to property schedulerGetDataChangeTimeout + * @param schedulerGetDataChangeTimeout value to be assigned to property schedulerGetDataChangeTimeout */ public void setSchedulerGetDataChangeTimeout(int schedulerGetDataChangeTimeout) { this.schedulerGetDataChangeTimeout = schedulerGetDataChangeTimeout; @@ -522,7 +571,7 @@ public int getSchedulerGetDataChangeFirstDelay() { /** * Setter method for property schedulerGetDataChangeFirstDelay. * - * @param schedulerGetDataChangeFirstDelay value to be assigned to property schedulerGetDataChangeFirstDelay + * @param schedulerGetDataChangeFirstDelay value to be assigned to property schedulerGetDataChangeFirstDelay */ public void setSchedulerGetDataChangeFirstDelay(int schedulerGetDataChangeFirstDelay) { this.schedulerGetDataChangeFirstDelay = schedulerGetDataChangeFirstDelay; @@ -541,7 +590,7 @@ public int getSchedulerGetDataChangeExpBackOffBound() { /** * Setter method for property schedulerGetDataChangeExpBackOffBound. * - * @param schedulerGetDataChangeExpBackOffBound value to be assigned to property schedulerGetDataChangeExpBackOffBound + * @param schedulerGetDataChangeExpBackOffBound value to be assigned to property schedulerGetDataChangeExpBackOffBound */ public void setSchedulerGetDataChangeExpBackOffBound(int schedulerGetDataChangeExpBackOffBound) { this.schedulerGetDataChangeExpBackOffBound = schedulerGetDataChangeExpBackOffBound; @@ -580,7 +629,7 @@ public String getRaftDataPath() { /** * Setter method for property raftDataPath. * - * @param raftDataPath value to be assigned to property raftDataPath + * @param raftDataPath value to be assigned to property raftDataPath */ public void setRaftDataPath(String raftDataPath) { this.raftDataPath = raftDataPath; @@ -594,9 +643,342 @@ public boolean isEnableMetrics() { /** * Setter method for property enableMetrics. * - * @param enableMetrics value to be assigned to property enableMetrics + * @param enableMetrics value to be assigned to property enableMetrics */ public void setEnableMetrics(boolean enableMetrics) { this.enableMetrics = enableMetrics; } + + /** + * Getter method for property RockDBCacheSize. + * + * @return property value of RockDBCacheSize + */ + public int getRockDBCacheSize() { + return rockDBCacheSize; + } + + /** + * Setter method for property RockDBCacheSize. + * + * @param rockDBCacheSize value to be assigned to property RockDBCacheSize + */ + public void setRockDBCacheSize(int rockDBCacheSize) { + this.rockDBCacheSize = rockDBCacheSize; + } + + /** + * Getter method for property heartbeatCheckExecutorMinSize. + * + * @return property value of heartbeatCheckExecutorMinSize + */ + @Override + public int getHeartbeatCheckExecutorMinSize() { + return heartbeatCheckExecutorMinSize; + } + + /** + * Getter method for property heartbeatCheckExecutorMaxSize. + * + * @return property value of heartbeatCheckExecutorMaxSize + */ + @Override + public int getHeartbeatCheckExecutorMaxSize() { + return heartbeatCheckExecutorMaxSize; + } + + /** + * Getter method for property heartbeatCheckExecutorQueueSize. + * + * @return property value of heartbeatCheckExecutorQueueSize + */ + @Override + public int getHeartbeatCheckExecutorQueueSize() { + return heartbeatCheckExecutorQueueSize; + } + + /** + * Getter method for property checkDataChangeExecutorMinSize. + * + * @return property value of checkDataChangeExecutorMinSize + */ + @Override + public int getCheckDataChangeExecutorMinSize() { + return checkDataChangeExecutorMinSize; + } + + /** + * Getter method for property checkDataChangeExecutorMaxSize. + * + * @return property value of checkDataChangeExecutorMaxSize + */ + @Override + public int getCheckDataChangeExecutorMaxSize() { + return checkDataChangeExecutorMaxSize; + } + + /** + * Getter method for property checkDataChangeExecutorQueueSize. + * + * @return property value of checkDataChangeExecutorQueueSize + */ + @Override + public int getCheckDataChangeExecutorQueueSize() { + return checkDataChangeExecutorQueueSize; + } + + /** + * Getter method for property getOtherDataCenterChangeExecutorMinSize. + * + * @return property value of getOtherDataCenterChangeExecutorMinSize + */ + @Override + public int getGetOtherDataCenterChangeExecutorMinSize() { + return getOtherDataCenterChangeExecutorMinSize; + } + + /** + * Getter method for property getOtherDataCenterChangeExecutorMaxSize. + * + * @return property value of getOtherDataCenterChangeExecutorMaxSize + */ + @Override + public int getGetOtherDataCenterChangeExecutorMaxSize() { + return getOtherDataCenterChangeExecutorMaxSize; + } + + /** + * Getter method for property getOtherDataCenterChangeExecutorQueueSize. + * + * @return property value of getOtherDataCenterChangeExecutorQueueSize + */ + @Override + public int getGetOtherDataCenterChangeExecutorQueueSize() { + return getOtherDataCenterChangeExecutorQueueSize; + } + + /** + * Getter method for property connectMetaServerExecutorMinSize. + * + * @return property value of connectMetaServerExecutorMinSize + */ + @Override + public int getConnectMetaServerExecutorMinSize() { + return connectMetaServerExecutorMinSize; + } + + /** + * Getter method for property connectMetaServerExecutorMaxSize. + * + * @return property value of connectMetaServerExecutorMaxSize + */ + @Override + public int getConnectMetaServerExecutorMaxSize() { + return connectMetaServerExecutorMaxSize; + } + + /** + * Getter method for property connectMetaServerExecutorQueueSize. + * + * @return property value of connectMetaServerExecutorQueueSize + */ + @Override + public int getConnectMetaServerExecutorQueueSize() { + return connectMetaServerExecutorQueueSize; + } + + /** + * Getter method for property checkNodeListChangePushExecutorMinSize. + * + * @return property value of checkNodeListChangePushExecutorMinSize + */ + @Override + public int getCheckNodeListChangePushExecutorMinSize() { + return checkNodeListChangePushExecutorMinSize; + } + + /** + * Getter method for property checkNodeListChangePushExecutorMaxSize. + * + * @return property value of checkNodeListChangePushExecutorMaxSize + */ + @Override + public int getCheckNodeListChangePushExecutorMaxSize() { + return checkNodeListChangePushExecutorMaxSize; + } + + /** + * Getter method for property checkNodeListChangePushExecutorQueueSize. + * + * @return property value of checkNodeListChangePushExecutorQueueSize + */ + @Override + public int getCheckNodeListChangePushExecutorQueueSize() { + return checkNodeListChangePushExecutorQueueSize; + } + + /** + * Getter method for property raftClientRefreshExecutorMinSize. + * + * @return property value of raftClientRefreshExecutorMinSize + */ + @Override + public int getRaftClientRefreshExecutorMinSize() { + return raftClientRefreshExecutorMinSize; + } + + /** + * Getter method for property raftClientRefreshExecutorMaxSize. + * + * @return property value of raftClientRefreshExecutorMaxSize + */ + @Override + public int getRaftClientRefreshExecutorMaxSize() { + return raftClientRefreshExecutorMaxSize; + } + + /** + * Getter method for property raftClientRefreshExecutorQueueSize. + * + * @return property value of raftClientRefreshExecutorQueueSize + */ + @Override + public int getRaftClientRefreshExecutorQueueSize() { + return raftClientRefreshExecutorQueueSize; + } + + /** + * Setter method for property metaSchedulerPoolSize . + * + * @param metaSchedulerPoolSize value to be assigned to property metaSchedulerPoolSize + */ + public void setMetaSchedulerPoolSize(int metaSchedulerPoolSize) { + this.metaSchedulerPoolSize = metaSchedulerPoolSize; + } + + @Override + public int getMetaSchedulerPoolSize() { + return metaSchedulerPoolSize; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } + + public int getDefaultRequestExecutorMinSize() { + return defaultRequestExecutorMinSize; + } + + public void setDefaultRequestExecutorMinSize(int defaultRequestExecutorMinSize) { + this.defaultRequestExecutorMinSize = defaultRequestExecutorMinSize; + } + + public int getDefaultRequestExecutorMaxSize() { + return defaultRequestExecutorMaxSize; + } + + public void setDefaultRequestExecutorMaxSize(int defaultRequestExecutorMaxSize) { + this.defaultRequestExecutorMaxSize = defaultRequestExecutorMaxSize; + } + + public int getDefaultRequestExecutorQueueSize() { + return defaultRequestExecutorQueueSize; + } + + public void setDefaultRequestExecutorQueueSize(int defaultRequestExecutorQueueSize) { + this.defaultRequestExecutorQueueSize = defaultRequestExecutorQueueSize; + } + + public int getRaftExecutorMinSize() { + return raftExecutorMinSize; + } + + public void setRaftExecutorMinSize(int raftExecutorMinSize) { + this.raftExecutorMinSize = raftExecutorMinSize; + } + + public int getRaftExecutorMaxSize() { + return raftExecutorMaxSize; + } + + public void setRaftExecutorMaxSize(int raftExecutorMaxSize) { + this.raftExecutorMaxSize = raftExecutorMaxSize; + } + + public int getRaftExecutorQueueSize() { + return raftExecutorQueueSize; + } + + public void setRaftExecutorQueueSize(int raftExecutorQueueSize) { + this.raftExecutorQueueSize = raftExecutorQueueSize; + } + + public int getRaftServerExecutorMinSize() { + return raftServerExecutorMinSize; + } + + public void setRaftServerExecutorMinSize(int raftServerExecutorMinSize) { + this.raftServerExecutorMinSize = raftServerExecutorMinSize; + } + + public int getRaftServerExecutorMaxSize() { + return raftServerExecutorMaxSize; + } + + public void setRaftServerExecutorMaxSize(int raftServerExecutorMaxSize) { + this.raftServerExecutorMaxSize = raftServerExecutorMaxSize; + } + + public int getRaftServerExecutorQueueSize() { + return raftServerExecutorQueueSize; + } + + public void setRaftServerExecutorQueueSize(int raftServerExecutorQueueSize) { + this.raftServerExecutorQueueSize = raftServerExecutorQueueSize; + } + + @Override + public int getRaftFsmExecutorMinSize() { + return raftFsmExecutorMinSize; + } + + public void setRaftFsmExecutorMinSize(int raftFsmExecutorMinSize) { + this.raftFsmExecutorMinSize = raftFsmExecutorMinSize; + } + + @Override + public int getRaftFsmExecutorMaxSize() { + return raftFsmExecutorMaxSize; + } + + public void setRaftFsmExecutorMaxSize(int raftFsmExecutorMaxSize) { + this.raftFsmExecutorMaxSize = raftFsmExecutorMaxSize; + } + + @Override + public int getRaftFsmExecutorQueueSize() { + return raftFsmExecutorQueueSize; + } + + public void setRaftFsmExecutorQueueSize(int raftFsmExecutorQueueSize) { + this.raftFsmExecutorQueueSize = raftFsmExecutorQueueSize; + } + + public int getRaftElectionTimeout() { + return raftElectionTimeout; + } + + public void setRaftElectionTimeout(int raftElectionTimeout) { + this.raftElectionTimeout = raftElectionTimeout; + } + + @Override + public double getSessionLoadbalanceThresholdRatio() { + return sessionLoadbalanceThresholdRatio; + } + + public void setSessionLoadbalanceThresholdRatio(double sessionLoadbalanceThresholdRatio) { + this.sessionLoadbalanceThresholdRatio = sessionLoadbalanceThresholdRatio; + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfiguration.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfiguration.java index 745881be0..9e2f90ca6 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfiguration.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerConfiguration.java @@ -16,6 +16,25 @@ */ package com.alipay.sofa.registry.server.meta.bootstrap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.alipay.sofa.registry.server.meta.executor.MetaMetricsThreadPoolExecutor; +import com.alipay.sofa.registry.server.meta.remoting.handler.*; +import com.alipay.sofa.registry.server.meta.timertask.LogMetricsTask; +import com.alipay.sofa.registry.util.NamedThreadFactory; +import com.alipay.sofa.registry.server.meta.resource.*; +import org.glassfish.jersey.jackson.JacksonFeature; +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + import com.alipay.sofa.registry.jraft.service.PersistenceDataDBService; import com.alipay.sofa.registry.remoting.bolt.exchange.BoltExchange; import com.alipay.sofa.registry.remoting.exchange.Exchange; @@ -40,13 +59,6 @@ import com.alipay.sofa.registry.server.meta.remoting.connection.DataConnectionHandler; import com.alipay.sofa.registry.server.meta.remoting.connection.MetaConnectionHandler; import com.alipay.sofa.registry.server.meta.remoting.connection.SessionConnectionHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.AbstractServerHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.DataNodeHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.FetchProvideDataRequestHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.GetChangeListRequestHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.GetNodesRequestHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.ReNewNodesRequestHandler; -import com.alipay.sofa.registry.server.meta.remoting.handler.SessionNodeHandler; import com.alipay.sofa.registry.server.meta.repository.NodeConfirmStatusService; import com.alipay.sofa.registry.server.meta.repository.RepositoryService; import com.alipay.sofa.registry.server.meta.repository.VersionRepositoryService; @@ -57,12 +69,6 @@ import com.alipay.sofa.registry.server.meta.repository.service.SessionConfirmStatusService; import com.alipay.sofa.registry.server.meta.repository.service.SessionRepositoryService; import com.alipay.sofa.registry.server.meta.repository.service.SessionVersionRepositoryService; -import com.alipay.sofa.registry.server.meta.resource.DecisionModeResource; -import com.alipay.sofa.registry.server.meta.resource.HealthResource; -import com.alipay.sofa.registry.server.meta.resource.MetaDigestResource; -import com.alipay.sofa.registry.server.meta.resource.MetaStoreResource; -import com.alipay.sofa.registry.server.meta.resource.PersistentDataResource; -import com.alipay.sofa.registry.server.meta.resource.StopPushDataResource; import com.alipay.sofa.registry.server.meta.store.DataStoreService; import com.alipay.sofa.registry.server.meta.store.MetaStoreService; import com.alipay.sofa.registry.server.meta.store.SessionStoreService; @@ -76,16 +82,6 @@ import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; import com.alipay.sofa.registry.util.PropertySplitter; -import org.glassfish.jersey.jackson.JacksonFeature; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; - -import java.util.ArrayList; -import java.util.Collection; /** * @@ -228,7 +224,7 @@ public Collection sessionServerHandlers() { Collection list = new ArrayList<>(); list.add(sessionConnectionHandler()); list.add(sessionNodeHandler()); - list.add(reNewNodesRequestHandler()); + list.add(renewNodesRequestHandler()); list.add(getNodesRequestHandler()); list.add(fetchProvideDataRequestHandler()); return list; @@ -240,7 +236,8 @@ public Collection dataServerHandlers() { list.add(dataConnectionHandler()); list.add(getNodesRequestHandler()); list.add(dataNodeHandler()); - list.add(reNewNodesRequestHandler()); + list.add(renewNodesRequestHandler()); + list.add(fetchProvideDataRequestHandler()); return list; } @@ -248,7 +245,6 @@ public Collection dataServerHandlers() { public Collection metaServerHandlers() { Collection list = new ArrayList<>(); list.add(metaConnectionHandler()); - list.add(getChangeListRequestHandler()); list.add(getNodesRequestHandler()); return list; } @@ -268,11 +264,6 @@ public AbstractServerHandler metaConnectionHandler() { return new MetaConnectionHandler(); } - @Bean - public AbstractServerHandler getChangeListRequestHandler() { - return new GetChangeListRequestHandler(); - } - @Bean public AbstractServerHandler getNodesRequestHandler() { return new GetNodesRequestHandler(); @@ -284,8 +275,8 @@ public AbstractServerHandler sessionNodeHandler() { } @Bean - public AbstractServerHandler reNewNodesRequestHandler() { - return new ReNewNodesRequestHandler(); + public AbstractServerHandler renewNodesRequestHandler() { + return new RenewNodesRequestHandler(); } @Bean @@ -355,9 +346,25 @@ public MetaStoreResource metaStoreResource() { } @Bean + @ConditionalOnMissingBean public StopPushDataResource stopPushDataResource() { return new StopPushDataResource(); } + + @Bean + public BlacklistDataResource blacklistDataResource() { + return new BlacklistDataResource(); + } + + @Bean + public RenewSwitchResource renewSwitchResource() { + return new RenewSwitchResource(); + } + + @Bean + public SessionLoadbalanceResource sessionLoadbalanceSwitchResource() { + return new SessionLoadbalanceResource(); + } } @Configuration @@ -424,6 +431,17 @@ public ExecutorManager executorManager(MetaServerConfig metaServerConfig) { return new ExecutorManager(metaServerConfig); } + @Bean + public ThreadPoolExecutor defaultRequestExecutor(MetaServerConfig metaServerConfig) { + ThreadPoolExecutor defaultRequestExecutor = new MetaMetricsThreadPoolExecutor( + "DefaultRequestExecutor", metaServerConfig.getDefaultRequestExecutorMinSize(), + metaServerConfig.getDefaultRequestExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getDefaultRequestExecutorQueueSize()), + new NamedThreadFactory("DefaultRequestExecutor")); + defaultRequestExecutor.allowCoreThreadTimeOut(true); + return defaultRequestExecutor; + } + } @Configuration @@ -433,4 +451,14 @@ public DBService persistenceDataDBService() { return new PersistenceDataDBService(); } } + + @Configuration + public static class TimerTask { + + @Bean + public LogMetricsTask logMetricsTask() { + return new LogMetricsTask(); + + } + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerInitializerConfiguration.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerInitializerConfiguration.java index 1add54080..cca6532fc 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerInitializerConfiguration.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/bootstrap/MetaServerInitializerConfiguration.java @@ -16,12 +16,13 @@ */ package com.alipay.sofa.registry.server.meta.bootstrap; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; +import java.util.concurrent.atomic.AtomicBoolean; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.SmartLifecycle; -import java.util.concurrent.atomic.AtomicBoolean; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; /** * @@ -47,7 +48,7 @@ public boolean isAutoStartup() { public void start() { try { - metaServerBootstrap.doInitialized(); + metaServerBootstrap.start(); LOGGER.info("Started MetaServer"); MetaServerInitializerConfiguration.this.running.set(true); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/ExecutorManager.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/ExecutorManager.java index 044572398..bd9fd6aff 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/ExecutorManager.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/ExecutorManager.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.meta.executor; +import java.util.concurrent.*; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.registry.Registry; @@ -23,16 +27,8 @@ import com.alipay.sofa.registry.server.meta.remoting.RaftExchanger; import com.alipay.sofa.registry.task.scheduler.TimedSupervisorTask; import com.alipay.sofa.registry.util.NamedThreadFactory; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; /** - * * @author shangyu.wh * @version $Id: ExecutorManager.java, v 0.1 2018-01-16 15:51 shangyu.wh Exp $ */ @@ -65,6 +61,7 @@ public class ExecutorManager { /** * constructor + * * @param metaServerConfig */ public ExecutorManager(MetaServerConfig metaServerConfig) { @@ -73,81 +70,97 @@ public ExecutorManager(MetaServerConfig metaServerConfig) { public void init() { - scheduler = new ScheduledThreadPoolExecutor(6, new NamedThreadFactory("MetaScheduler")); - - heartbeatCheckExecutor = new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("MetaScheduler-HeartbeatCheck")); - - checkDataChangeExecutor = new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("MetaScheduler-CheckDataChange")); - - getOtherDataCenterChangeExecutor = new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory( + scheduler = new ScheduledThreadPoolExecutor(metaServerConfig.getMetaSchedulerPoolSize(), + new NamedThreadFactory("MetaScheduler")); + + heartbeatCheckExecutor = new ThreadPoolExecutor( + metaServerConfig.getHeartbeatCheckExecutorMinSize(), + metaServerConfig.getHeartbeatCheckExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getHeartbeatCheckExecutorQueueSize()), + new NamedThreadFactory("MetaScheduler-HeartbeatCheck")); + heartbeatCheckExecutor.allowCoreThreadTimeOut(true); + + checkDataChangeExecutor = new ThreadPoolExecutor( + metaServerConfig.getCheckDataChangeExecutorMinSize(), + metaServerConfig.getCheckDataChangeExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getCheckDataChangeExecutorQueueSize()), + new NamedThreadFactory("MetaScheduler-CheckDataChange")); + checkDataChangeExecutor.allowCoreThreadTimeOut(true); + + getOtherDataCenterChangeExecutor = new ThreadPoolExecutor( + metaServerConfig.getGetOtherDataCenterChangeExecutorMinSize(), + metaServerConfig.getGetOtherDataCenterChangeExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig + .getGetOtherDataCenterChangeExecutorQueueSize()), new NamedThreadFactory( "MetaScheduler-GetOtherDataCenterChange")); + getOtherDataCenterChangeExecutor.allowCoreThreadTimeOut(true); + + connectMetaServerExecutor = new ThreadPoolExecutor( + metaServerConfig.getConnectMetaServerExecutorMinSize(), + metaServerConfig.getConnectMetaServerExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getConnectMetaServerExecutorQueueSize()), + new NamedThreadFactory("MetaScheduler-ConnectMetaServer")); + connectMetaServerExecutor.allowCoreThreadTimeOut(true); + + checkNodeListChangePushExecutor = new ThreadPoolExecutor( + metaServerConfig.getCheckNodeListChangePushExecutorMinSize(), + metaServerConfig.getCheckNodeListChangePushExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getCheckDataChangeExecutorQueueSize()), + new NamedThreadFactory("MetaScheduler-CheckNodeListChangePush")); + checkNodeListChangePushExecutor.allowCoreThreadTimeOut(true); + + raftClientRefreshExecutor = new ThreadPoolExecutor( + metaServerConfig.getRaftClientRefreshExecutorMinSize(), + metaServerConfig.getRaftClientRefreshExecutorMaxSize(), 300, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getRaftClientRefreshExecutorQueueSize()), + new NamedThreadFactory("MetaScheduler-RaftClientRefresh")); + raftClientRefreshExecutor.allowCoreThreadTimeOut(true); - connectMetaServerExecutor = new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("MetaScheduler-ConnectMetaServer")); - - checkNodeListChangePushExecutor = new ThreadPoolExecutor(1, 4, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory( - "MetaScheduler-CheckNodeListChangePush")); - - raftClientRefreshExecutor = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("MetaScheduler-RaftClientRefresh")); } public void startScheduler() { init(); - scheduler.schedule( - new TimedSupervisorTask("HeartbeatCheck", scheduler, heartbeatCheckExecutor, + scheduler.schedule(new TimedSupervisorTask("HeartbeatCheck", scheduler, heartbeatCheckExecutor, metaServerConfig.getSchedulerHeartbeatTimeout(), TimeUnit.SECONDS, - metaServerConfig.getSchedulerHeartbeatExpBackOffBound(), - () -> metaServerRegistry.evict()), + metaServerConfig.getSchedulerHeartbeatExpBackOffBound(), () -> metaServerRegistry.evict()), metaServerConfig.getSchedulerHeartbeatFirstDelay(), TimeUnit.SECONDS); - scheduler.schedule(new TimedSupervisorTask("GetOtherDataCenterChange", scheduler, - getOtherDataCenterChangeExecutor, metaServerConfig.getSchedulerGetDataChangeTimeout(), - TimeUnit.SECONDS, metaServerConfig.getSchedulerGetDataChangeExpBackOffBound(), - () -> { - metaServerRegistry.getOtherDataCenterNodeAndUpdate(NodeType.DATA); - metaServerRegistry.getOtherDataCenterNodeAndUpdate(NodeType.META); - }), - metaServerConfig.getSchedulerGetDataChangeFirstDelay(), TimeUnit.SECONDS); - scheduler.schedule( - new TimedSupervisorTask("ConnectMetaServer", scheduler, connectMetaServerExecutor, + new TimedSupervisorTask("GetOtherDataCenterChange", scheduler, getOtherDataCenterChangeExecutor, + metaServerConfig.getSchedulerGetDataChangeTimeout(), TimeUnit.SECONDS, + metaServerConfig.getSchedulerGetDataChangeExpBackOffBound(), () -> { + metaServerRegistry.getOtherDataCenterNodeAndUpdate(NodeType.DATA); + metaServerRegistry.getOtherDataCenterNodeAndUpdate(NodeType.META); + }), metaServerConfig.getSchedulerGetDataChangeFirstDelay(), TimeUnit.SECONDS); + + scheduler.schedule(new TimedSupervisorTask("ConnectMetaServer", scheduler, connectMetaServerExecutor, metaServerConfig.getSchedulerConnectMetaServerTimeout(), TimeUnit.SECONDS, metaServerConfig.getSchedulerConnectMetaServerExpBackOffBound(), - () -> metaClientExchanger.connectServer()), - metaServerConfig.getSchedulerConnectMetaServerFirstDelay(), TimeUnit.SECONDS); + () -> metaClientExchanger.connectServer()), metaServerConfig.getSchedulerConnectMetaServerFirstDelay(), + TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("CheckSessionNodeListChangePush", scheduler, - checkNodeListChangePushExecutor, + new TimedSupervisorTask("CheckSessionNodeListChangePush", scheduler, checkNodeListChangePushExecutor, metaServerConfig.getSchedulerCheckNodeListChangePushTimeout(), TimeUnit.SECONDS, metaServerConfig.getSchedulerCheckNodeListChangePushExpBackOffBound(), () -> metaServerRegistry.pushNodeListChange(NodeType.SESSION)), metaServerConfig.getSchedulerCheckNodeListChangePushFirstDelay(), TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("CheckDataNodeListChangePush", scheduler, - checkNodeListChangePushExecutor, + new TimedSupervisorTask("CheckDataNodeListChangePush", scheduler, checkNodeListChangePushExecutor, metaServerConfig.getSchedulerCheckNodeListChangePushTimeout(), TimeUnit.SECONDS, metaServerConfig.getSchedulerCheckNodeListChangePushExpBackOffBound(), () -> metaServerRegistry.pushNodeListChange(NodeType.DATA)), metaServerConfig.getSchedulerCheckNodeListChangePushFirstDelay(), TimeUnit.SECONDS); - scheduler.schedule( - new TimedSupervisorTask("RaftClientRefresh", scheduler, - raftClientRefreshExecutor, + scheduler.schedule(new TimedSupervisorTask("RaftClientRefresh", scheduler, raftClientRefreshExecutor, metaServerConfig.getSchedulerCheckNodeListChangePushTimeout(), TimeUnit.SECONDS, metaServerConfig.getSchedulerCheckNodeListChangePushExpBackOffBound(), () -> raftExchanger.refreshRaftClient()), metaServerConfig.getSchedulerCheckNodeListChangePushFirstDelay(), TimeUnit.SECONDS); - - } + } public void stopScheduler() { if (scheduler != null && !scheduler.isShutdown()) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionThreadPoolExecutor.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/MetaMetricsThreadPoolExecutor.java similarity index 53% rename from server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionThreadPoolExecutor.java rename to server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/MetaMetricsThreadPoolExecutor.java index 18652a22c..a81ed2bb8 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionThreadPoolExecutor.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/executor/MetaMetricsThreadPoolExecutor.java @@ -14,59 +14,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alipay.sofa.registry.server.session.scheduler; +package com.alipay.sofa.registry.server.meta.executor; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.metrics.TaskMetrics; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; -/** - * - * @author shangyu.wh - * @version $Id: ThreadPoolExecutorSession.java, v 0.1 2018-10-11 19:07 shangyu.wh Exp $ - */ -public class SessionThreadPoolExecutor extends ThreadPoolExecutor { - - private static final Logger LOGGER = LoggerFactory.getLogger(SessionThreadPoolExecutor.class); +public class MetaMetricsThreadPoolExecutor extends ThreadPoolExecutor { + private static final Logger LOGGER = LoggerFactory + .getLogger(MetaMetricsThreadPoolExecutor.class); private String executorName; - public SessionThreadPoolExecutor(String executorName, int corePoolSize, int maximumPoolSize, - long keepAliveTime, TimeUnit unit, - BlockingQueue workQueue, ThreadFactory threadFactory) { + public MetaMetricsThreadPoolExecutor(String executorName, int corePoolSize, + int maximumPoolSize, long keepAliveTime, TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); - this.executorName = executorName; + this.executorName = "Meta-" + executorName; registerTaskMetrics(); } - private void registerTaskMetrics() { - - TaskMetrics.getInstance().registerThreadExecutor(executorName, this); - } - @Override public String toString() { + return super.toString() + executorName; + } - return (new StringBuilder(executorName).append(" ").append(super.toString())).toString(); + private void registerTaskMetrics() { + TaskMetrics.getInstance().registerThreadExecutor(executorName, this); } @Override public void execute(Runnable command) { - try { super.execute(command); - } catch (Exception e) { - if (e instanceof RejectedExecutionException) { - LOGGER.error("Processor session executor {} Rejected Execution!command {}", this, - command.getClass(), e); - } - throw e; + } catch (RejectedExecutionException e) { + LOGGER.error("Processor meta executor {} Rejected Execution!command {}", this, + command.getClass(), e); } } -} \ No newline at end of file +} diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/DataNodeChangePushTaskListener.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/DataNodeChangePushTaskListener.java index a146d7721..de336874f 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/DataNodeChangePushTaskListener.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/DataNodeChangePushTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.task.Constant; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -57,8 +58,8 @@ public DataNodeChangePushTaskListener(TaskProcessor dataNodeSingleTaskProcessor) } @Override - public boolean support(TaskEvent event) { - return TaskType.DATA_NODE_CHANGE_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.DATA_NODE_CHANGE_PUSH_TASK; } @Override diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/PersistenceDataChangeNotifyTaskListener.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/PersistenceDataChangeNotifyTaskListener.java index f2e5714ed..b72725e4f 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/PersistenceDataChangeNotifyTaskListener.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/PersistenceDataChangeNotifyTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.task.MetaServerTask; import com.alipay.sofa.registry.server.meta.task.PersistenceDataChangeNotifyTask; @@ -25,7 +27,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -49,8 +50,8 @@ public PersistenceDataChangeNotifyTaskListener(TaskProcessor sessionNodeSingleTa } @Override - public boolean support(TaskEvent event) { - return TaskType.PERSISTENCE_DATA_CHANGE_NOTIFY_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.PERSISTENCE_DATA_CHANGE_NOTIFY_TASK; } @Override diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/ReceiveStatusConfirmNotifyTaskListener.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/ReceiveStatusConfirmNotifyTaskListener.java index 66b8dbe6b..d910d247f 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/ReceiveStatusConfirmNotifyTaskListener.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/ReceiveStatusConfirmNotifyTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.node.DataNodeService; import com.alipay.sofa.registry.server.meta.node.NodeService; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -53,8 +54,8 @@ public ReceiveStatusConfirmNotifyTaskListener(TaskProcessor dataNodeSingleTaskPr } @Override - public boolean support(TaskEvent event) { - return TaskType.RECEIVE_STATUS_CONFIRM_NOTIFY_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.RECEIVE_STATUS_CONFIRM_NOTIFY_TASK; } @Override diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/SessionNodeChangePushTaskListener.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/SessionNodeChangePushTaskListener.java index 24dd13e3a..fb7edd296 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/SessionNodeChangePushTaskListener.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/listener/SessionNodeChangePushTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.task.MetaServerTask; import com.alipay.sofa.registry.server.meta.task.SessionNodeChangePushTask; @@ -25,7 +27,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -49,8 +50,8 @@ public SessionNodeChangePushTaskListener(TaskProcessor sessionNodeSingleTaskProc } @Override - public boolean support(TaskEvent event) { - return TaskType.SESSION_NODE_CHANGE_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.SESSION_NODE_CHANGE_PUSH_TASK; } @Override diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/DataNodeService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/DataNodeService.java index 9fc3efccc..10f77b834 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/DataNodeService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/DataNodeService.java @@ -18,6 +18,7 @@ import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; +import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; import com.alipay.sofa.registry.common.model.metaserver.StatusConfirmRequest; import java.util.Map; @@ -33,4 +34,6 @@ void pushDataNodes(NodeChangeResult nodeChangeResult, Map targ boolean confirm, String confirmNodeIp); void notifyStatusConfirm(StatusConfirmRequest statusConfirmRequest); + + void notifyProvideDataChange(NotifyProvideDataChange notifyProvideDataChange); } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/SessionNodeService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/SessionNodeService.java index 102f368a6..006783f62 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/SessionNodeService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/SessionNodeService.java @@ -16,6 +16,7 @@ */ package com.alipay.sofa.registry.server.meta.node; +import com.alipay.sofa.registry.common.model.metaserver.LoadbalanceMetrics; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; import com.alipay.sofa.registry.common.model.metaserver.SessionNode; @@ -35,4 +36,8 @@ void pushSessions(NodeChangeResult nodeChangeResult, Map ta void pushDataNodes(NodeChangeResult nodeChangeResult); void notifyProvideDataChange(NotifyProvideDataChange notifyProvideDataChange); + + Map> fetchLoadbalanceMetrics(); + + void configureLoadbalance(Map> maxConnectionsMap); } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/DataNodeServiceImpl.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/DataNodeServiceImpl.java index d7bb82df1..a8300f1b5 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/DataNodeServiceImpl.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/DataNodeServiceImpl.java @@ -21,6 +21,7 @@ import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; +import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; import com.alipay.sofa.registry.common.model.metaserver.StatusConfirmRequest; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; @@ -134,7 +135,7 @@ public URL getRequestUrl() { } } catch (RequestException e) { - LOGGER.error("Push dataNode list error! " + e.getRequestMessage(), e); + LOGGER.error("Push dataNode list error! " + e.getMessage(), e); exceptions.add(e); } catch (RuntimeException e) { LOGGER.error("Push dataNode list runtime error! ", e); @@ -178,8 +179,56 @@ public URL getRequestUrl() { } } catch (RequestException e) { - LOGGER.error("Notify status confirm error! " + e.getRequestMessage(), e); - throw new RuntimeException("Notify status confirm error! ", e); + throw new RuntimeException("Notify status confirm error: " + e.getMessage(), e); + } + } + + @Override + public void notifyProvideDataChange(NotifyProvideDataChange notifyProvideDataChange) { + + NodeConnectManager nodeConnectManager = getNodeConnectManager(); + Collection connections = nodeConnectManager.getConnections(null); + + if (connections == null || connections.isEmpty()) { + LOGGER.error("Push dataNode list error! No data node connected!"); + throw new RuntimeException("Push dataNode list error! No data node connected!"); + } + + // add register confirm + StoreService storeService = ServiceFactory.getStoreService(NodeType.DATA); + Map dataNodes = storeService.getNodes(); + + if (dataNodes == null || dataNodes.isEmpty()) { + LOGGER.error("Push dataNode list error! No data node registered!"); + throw new RuntimeException("Push dataNode list error! No data node registered!"); + } + + for (InetSocketAddress connection : connections) { + + if (!dataNodes.keySet().contains(connection.getAddress().getHostAddress())) { + continue; + } + + try { + Request request = new Request() { + + @Override + public NotifyProvideDataChange getRequestBody() { + return notifyProvideDataChange; + } + + @Override + public URL getRequestUrl() { + return new URL(connection); + } + }; + + dataNodeExchanger.request(request); + + } catch (RequestException e) { + throw new RuntimeException("Notify provide data change to dataServer error: " + + e.getMessage(), e); + } } } diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/MetaNodeServiceImpl.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/MetaNodeServiceImpl.java index e0aead477..255287c97 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/MetaNodeServiceImpl.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/MetaNodeServiceImpl.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.node.impl; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.GetChangeListRequest; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.meta.node.MetaNodeService; import com.alipay.sofa.registry.server.meta.remoting.MetaClientExchanger; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -69,9 +70,8 @@ public URL getRequestUrl() { } } catch (RequestException e) { - LOGGER.error("MetaNodeService get DataCenter Nodes error! " + e.getRequestMessage(), e); throw new RuntimeException("MetaNodeService get DataCenter Nodes error! " - + e.getRequestMessage(), e); + + e.getMessage(), e); } } diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/SessionNodeServiceImpl.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/SessionNodeServiceImpl.java index 9e3da8ab4..d523fef6a 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/SessionNodeServiceImpl.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/node/impl/SessionNodeServiceImpl.java @@ -16,10 +16,20 @@ */ package com.alipay.sofa.registry.server.meta.node.impl; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.alipay.sofa.registry.common.model.metaserver.*; +import com.alipay.sofa.registry.remoting.exchange.message.Response; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; -import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; -import com.alipay.sofa.registry.common.model.metaserver.SessionNode; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -32,14 +42,8 @@ import com.alipay.sofa.registry.server.meta.remoting.connection.NodeConnectManager; import com.alipay.sofa.registry.server.meta.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.meta.store.StoreService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.Map; /** - * * @author shangyu.wh * @version $Id: SessionNodeServiceImpl.java, v 0.1 2018-01-15 17:18 shangyu.wh Exp $ */ @@ -113,8 +117,7 @@ public URL getRequestUrl() { confirmNodeIp); } catch (RequestException e) { - LOGGER.error("Push sessionNode list error! " + e.getRequestMessage(), e); - throw new RuntimeException("SessionNodeService push sessionNode list error!"); + throw new RuntimeException("Push sessionNode list error: " + e.getMessage(), e); } } @@ -163,8 +166,7 @@ public URL getRequestUrl() { sessionNodeExchanger.request(nodeChangeRequestRequest); } catch (RequestException e) { - LOGGER.error("Push sessionNode list error! " + e.getRequestMessage(), e); - throw new RuntimeException("SessionNodeService push dataNode list error!"); + throw new RuntimeException("Push sessionNode list error: " + e.getMessage(), e); } } } @@ -212,8 +214,88 @@ public URL getRequestUrl() { sessionNodeExchanger.request(request); } catch (RequestException e) { - LOGGER.error("Notify provide data change error! " + e.getRequestMessage(), e); - throw new RuntimeException("Notify provide data change error!"); + throw new RuntimeException("Notify provide data change error: " + e.getMessage(), e); + } + } + } + + @Override + public Map> fetchLoadbalanceMetrics() { + Map sessionNodes = ServiceFactory.getStoreService(NodeType.SESSION) + .getNodes(); + Map connectionsMap = getNodeConnectManager().getConnections(null) + .stream() + .collect(Collectors.toMap(c -> c.getAddress().getHostAddress(), Function.identity())); + Map> zones = new HashMap<>(); + Set failedZones = new HashSet<>(); + for (Map.Entry entry : sessionNodes.entrySet()) { + String ipAddress = entry.getKey(); + SessionNode node = entry.getValue(); + InetSocketAddress connection = connectionsMap.get(ipAddress); + if (connection == null) { + failedZones.add(node.getRegionId()); + LOGGER.warn("node {} has no connection, skip fetch loadbalance metrics", node); + continue; + } + Map zone = zones.computeIfAbsent(node.getRegionId(), + k -> new HashMap<>()); + try { + Request metricsRequest = new Request() { + @Override + public GetLoadbalanceMetricsRequest getRequestBody() { + return new GetLoadbalanceMetricsRequest(); + } + + @Override + public URL getRequestUrl() { + return new URL(connection); + } + }; + Response response = sessionNodeExchanger.request(metricsRequest); + LoadbalanceMetrics metrics = (LoadbalanceMetrics) response.getResult(); + zone.put(ipAddress, metrics); + } catch (RequestException e) { + LOGGER.warn("node {} failed to fetch load balance metrics: {}", node, + e.getMessage()); + failedZones.add(node.getRegionId()); + } + } + for (String failedZone : failedZones) { + zones.remove(failedZone); + } + return zones; + } + + @Override + public void configureLoadbalance(Map> zonesMaxConnectionsMap) { + Map connectionsMap = getNodeConnectManager().getConnections(null) + .stream() + .collect(Collectors.toMap(c -> c.getAddress().getHostAddress(), Function.identity())); + + for (String zoneName : zonesMaxConnectionsMap.keySet()) { + Map maxConnectionsMap = zonesMaxConnectionsMap.get(zoneName); + for (String ipAddress : maxConnectionsMap.keySet()) { + InetSocketAddress connection = connectionsMap.get(ipAddress); + if (connection == null) { + continue; + } + Request request = new Request() { + @Override + public ConfigureLoadbalanceRequest getRequestBody() { + return new ConfigureLoadbalanceRequest(maxConnectionsMap.get(ipAddress)); + } + + @Override + public URL getRequestUrl() { + return new URL(connection); + } + }; + try { + sessionNodeExchanger.request(request); + } catch (RequestException e) { + LOGGER.warn("configure node {} load balance failed: {}", connection, + e.getMessage()); + } } } } @@ -227,4 +309,5 @@ private NodeConnectManager getNodeConnectManager() { return (NodeConnectManager) sessionConnectionHandler; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/MetaServerRegistry.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/MetaServerRegistry.java index 589c5ce4c..db03867e9 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/MetaServerRegistry.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/MetaServerRegistry.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.meta.registry; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; @@ -23,13 +27,12 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.meta.bootstrap.ServiceFactory; +import com.alipay.sofa.registry.server.meta.store.SessionStoreService; import com.alipay.sofa.registry.server.meta.store.StoreService; -import java.util.Collection; -import java.util.List; - /** * factory func to operate StoreService + * * @author shangyu.wh * @version $Id: MetaServerRegistry.java, v 0.1 2018-01-11 21:38 shangyu.wh Exp $ */ @@ -73,9 +76,9 @@ public void evict() { } @Override - public void reNew(Node node, int duration) { + public void renew(Node node, int duration) { StoreService storeService = ServiceFactory.getStoreService(node.getNodeType()); - storeService.reNew(node, duration); + storeService.renew(node, duration); } @Override @@ -103,4 +106,11 @@ public void pushNodeListChange(NodeType nodeType) { storeService.pushNodeListChange(); } } + + @Override + public Map>> sessionLoadbalance(int maxDisconnect) { + SessionStoreService storeService = (SessionStoreService) ServiceFactory + .getStoreService(NodeType.SESSION); + return storeService.sessionLoadbalance(maxDisconnect); + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/Registry.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/Registry.java index 2bda79a84..233e0eeff 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/Registry.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/registry/Registry.java @@ -16,15 +16,15 @@ */ package com.alipay.sofa.registry.server.meta.registry; +import java.util.List; +import java.util.Map; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import java.util.List; - /** - * * @author shangyu.wh * @version $Id: Registry.java, v 0.1 2018-01-11 17:18 shangyu.wh Exp $ */ @@ -41,8 +41,9 @@ public interface Registry { /** * register new node * one node unique id by connectId "ip:port" - * @return return dataList + * * @param node + * @return return dataList */ NodeChangeResult register(T node); @@ -60,18 +61,21 @@ public interface Registry { /** * renew node expire time + * * @param node */ - void reNew(T node, int duration); + void renew(T node, int duration); /** * get other dataCenter Nodes change scheduled + * * @param nodeType */ void getOtherDataCenterNodeAndUpdate(NodeType nodeType); /** * get DataCenter Nodes list contains version + * * @param nodeType * @return */ @@ -79,6 +83,7 @@ public interface Registry { /** * get all dataCenter Node list by NodeType + * * @param nodeType * @return */ @@ -88,4 +93,9 @@ public interface Registry { * push node change result */ void pushNodeListChange(NodeType nodeType); + + /** + * fetch session metrics and config load balancer + */ + Map>> sessionLoadbalance(int maxDisconnect); } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/DataNodeExchanger.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/DataNodeExchanger.java index 1ea35b9b2..c7c426e6b 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/DataNodeExchanger.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/DataNodeExchanger.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.remoting; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -28,7 +30,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Request; import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -59,7 +60,7 @@ public Response request(Request request) throws RequestException { if (channel != null && channel.isConnected()) { final Object result = dataServer.sendSync(channel, request.getRequestBody(), - metaServerConfig.getDataNodeExchangeTimeout()); + request.getTimeout() != null ? request.getTimeout() : metaServerConfig.getDataNodeExchangeTimeout()); response = () -> result; } else { LOGGER.error("DataNode Exchanger get channel error! channel with url:" + url diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaClientExchanger.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaClientExchanger.java index 4c48f713e..0c7b66433 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaClientExchanger.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaClientExchanger.java @@ -16,13 +16,22 @@ */ package com.alipay.sofa.registry.server.meta.remoting; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.Client; import com.alipay.sofa.registry.remoting.exchange.Exchange; @@ -34,15 +43,6 @@ import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; import com.alipay.sofa.registry.server.meta.bootstrap.ServiceFactory; import com.alipay.sofa.registry.server.meta.store.StoreService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; /** * @@ -82,14 +82,8 @@ public Response request(Request request) throws RequestException { metaClient = boltExchange.connect(Exchange.META_SERVER_TYPE, url, new ChannelHandler[0]); } - Channel channel = metaClient.getChannel(url); - if (channel == null) { - LOGGER.warn("MetaClient Exchanger get channel {} error or disconnected!", url); - channel = metaClient.connect(url); - } - - final Object result = metaClient.sendSync(channel, request.getRequestBody(), - metaServerConfig.getMetaNodeExchangeTimeout()); + final Object result = metaClient.sendSync(url, request.getRequestBody(), + request.getTimeout() != null ? request.getTimeout() : metaServerConfig.getMetaNodeExchangeTimeout()); response = () -> result; } catch (Exception e) { LOGGER.error("MetaClient Exchanger request data error!", e); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaServerExchanger.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaServerExchanger.java index 93c0857ee..2c5d908ee 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaServerExchanger.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/MetaServerExchanger.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.remoting; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -28,7 +30,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Request; import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -59,7 +60,7 @@ public Response request(Request request) throws RequestException { if (channel != null && channel.isConnected()) { final Object result = metaServer.sendSync(channel, request.getRequestBody(), - metaServerConfig.getDataNodeExchangeTimeout()); + request.getTimeout() != null ? request.getTimeout() : metaServerConfig.getDataNodeExchangeTimeout()); response = () -> result; } else { LOGGER.error("MetaServer Exchanger get channel error! channel with url:" diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/RaftExchanger.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/RaftExchanger.java index 6a35b4b7a..72b4cbd5b 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/RaftExchanger.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/RaftExchanger.java @@ -16,6 +16,23 @@ */ package com.alipay.sofa.registry.server.meta.remoting; +import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import com.alipay.remoting.ProtocolCode; +import com.alipay.remoting.ProtocolManager; +import com.alipay.remoting.rpc.protocol.RpcProtocol; +import com.alipay.sofa.jraft.util.ThreadPoolMetricSet; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.alipay.sofa.registry.server.meta.executor.MetaMetricsThreadPoolExecutor; +import com.alipay.sofa.registry.util.NamedThreadFactory; +import com.codahale.metrics.MetricRegistry; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.jraft.CliService; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.conf.Configuration; @@ -23,7 +40,7 @@ import com.alipay.sofa.jraft.core.NodeImpl; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.CliOptions; -import com.alipay.sofa.jraft.rpc.impl.AbstractBoltClientService; +import com.alipay.sofa.jraft.rpc.impl.AbstractClientService; import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.jraft.bootstrap.RaftClient; @@ -38,17 +55,8 @@ import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; import com.alipay.sofa.registry.server.meta.executor.ExecutorManager; import com.alipay.sofa.registry.server.meta.registry.Registry; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; /** - * * @author shangyu.wh * @version $Id: RaftExchanger.java, v 0.1 2018-05-22 15:13 shangyu.wh Exp $ */ @@ -58,12 +66,17 @@ public class RaftExchanger { private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("META-JRAFT-METRICS"); + private static final Logger LOGGER_START = LoggerFactory.getLogger("META-START-LOGS"); + @Autowired private MetaServerConfig metaServerConfig; @Autowired private NodeConfig nodeConfig; + @Autowired + private ThreadPoolExecutor defaultRequestExecutor; + @Autowired private Registry metaServerRegistry; @@ -79,6 +92,7 @@ public class RaftExchanger { /** * Start Raft server + * * @param executorManager */ public void startRaftServer(final ExecutorManager executorManager) { @@ -93,20 +107,21 @@ public void startRaftServer(final ExecutorManager executorManager) { raftServer.setLeaderProcessListener(new LeaderProcessListener() { @Override public void startProcess() { - LOGGER.info("Start leader process..."); + LOGGER_START.info("Start leader process..."); executorManager.startScheduler(); - LOGGER.info("Initialize server scheduler success!"); + LOGGER_START.info("Initialize server scheduler success!"); PeerId leader = new PeerId(NetUtil.getLocalAddress().getHostAddress(), metaServerConfig.getRaftServerPort()); - raftServer.sendNotify(leader, "leader"); + // refer: https://github.com/sofastack/sofa-registry/issues/30 registerCurrentNode(); + raftServer.sendNotify(leader, "leader"); } @Override public void stopProcess() { - LOGGER.info("Stop leader process..."); + LOGGER_START.info("Stop leader process..."); executorManager.stopScheduler(); - LOGGER.info("Stop server scheduler success!"); + LOGGER_START.info("Stop server scheduler success!"); PeerId leader = new PeerId(NetUtil.getLocalAddress().getHostAddress(), metaServerConfig.getRaftServerPort()); raftServer.sendNotify(leader, "leader"); @@ -116,27 +131,56 @@ public void stopProcess() { raftServer.setFollowerProcessListener(new FollowerProcessListener() { @Override public void startProcess(PeerId leader) { - LOGGER.info("Start follower process leader {}...", leader); - raftServer.sendNotify(leader, "follower"); + LOGGER_START.info("Start follower process leader {}...", leader); + // refer: https://github.com/sofastack/sofa-registry/issues/31 + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + LOGGER_START.error(e.getMessage(), e); + } registerCurrentNode(); + raftServer.sendNotify(leader, "follower"); } @Override public void stopProcess(PeerId leader) { - LOGGER.info("Stop follower process leader {}...", leader); + LOGGER_START.info("Stop follower process leader {}...", leader); raftServer.sendNotify(leader, "follower"); } }); - + ThreadPoolExecutor raftExecutor = new MetaMetricsThreadPoolExecutor("RaftExecutor", + metaServerConfig.getRaftExecutorMinSize(), + metaServerConfig.getRaftExecutorMaxSize(), 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getRaftExecutorQueueSize()), + new NamedThreadFactory("RaftExecutor", true)); + + ThreadPoolExecutor raftServerExecutor = new MetaMetricsThreadPoolExecutor( + "RaftServerExecutor", metaServerConfig.getRaftServerExecutorMinSize(), + metaServerConfig.getRaftServerExecutorMaxSize(), 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getRaftServerExecutorQueueSize()), + new NamedThreadFactory("RaftServerExecutor", true)); + + ThreadPoolExecutor fsmExecutor = new MetaMetricsThreadPoolExecutor( + "RaftFsmExecutor", metaServerConfig.getRaftFsmExecutorMinSize(), + metaServerConfig.getRaftFsmExecutorMaxSize(), 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(metaServerConfig.getRaftFsmExecutorQueueSize()), + new NamedThreadFactory("RaftFsmExecutor", true)); + + raftServer.setRaftExecutor(raftExecutor); + raftServer.setRaftServerExecutor(raftServerExecutor); + raftServer.setFsmExecutor(fsmExecutor); RaftServerConfig raftServerConfig = new RaftServerConfig(); raftServerConfig.setMetricsLogger(METRICS_LOGGER); raftServerConfig.setEnableMetrics(metaServerConfig.isEnableMetrics()); - - raftServer.start(raftServerConfig); + raftServerConfig.setElectionTimeoutMs(metaServerConfig.getRaftElectionTimeout()); + if (metaServerConfig.getRockDBCacheSize() > 0) { + raftServerConfig.setRockDBCacheSize(metaServerConfig.getRockDBCacheSize()); + } + raftServer.start(raftServerConfig, defaultRequestExecutor); } } catch (Exception e) { serverStart.set(false); - LOGGER.error("Start raft server error!", e); + LOGGER_START.error("Start raft server error!", e); throw new RuntimeException("Start raft server error!", e); } } @@ -150,17 +194,19 @@ public void startRaftClient() { String serverConf = getServerConfig(); if (raftServer != null && raftServer.getNode() != null) { //TODO this cannot be invoke,because RaftAnnotationBeanPostProcessor.getProxy will start first - raftClient = new RaftClient(getGroup(), serverConf, - (AbstractBoltClientService) (((NodeImpl) raftServer.getNode()) - .getRpcService())); + raftClient = new RaftClient( + getGroup(), + serverConf, + (AbstractClientService) (((NodeImpl) raftServer.getNode()).getRpcService()), + defaultRequestExecutor); } else { - raftClient = new RaftClient(getGroup(), serverConf); + raftClient = new RaftClient(getGroup(), serverConf, defaultRequestExecutor); } raftClient.start(); } } catch (Exception e) { clientStart.set(false); - LOGGER.error("Start raft client error!", e); + LOGGER_START.error("Start raft client error!", e); throw new RuntimeException("Start raft client error!", e); } } @@ -174,7 +220,7 @@ public void startCliService() { cliService = new CliServiceImpl(); cliService.init(new CliOptions()); } catch (Exception e) { - LOGGER.error("Start raft cliService error!", e); + LOGGER_START.error("Start raft cliService error!", e); throw new RuntimeException("Start raft cliService error!", e); } } @@ -190,7 +236,7 @@ private void registerCurrentNode() { metaServerRegistry.register(new MetaNode(new URL(ip, 0), nodeConfig .getLocalDataCenter())); } else { - LOGGER.error( + LOGGER_START.error( "Register CurrentNode fail!meta node list config not contains current ip {}", ip); throw new RuntimeException( @@ -262,6 +308,7 @@ public void resetPeer(List ipAddressList) { /** * api for remove meta node + * * @param ipAddress */ public void removePeer(String ipAddress) { @@ -327,7 +374,7 @@ private String getServerConfig() { Set ips = nodeConfig.getDataCenterMetaServers(nodeConfig.getLocalDataCenter()); if (ips != null && !ips.isEmpty()) { ret = ips.stream().map(ip -> ip + ":" + metaServerConfig.getRaftServerPort()) - .collect(Collectors.joining(",")); + .collect(Collectors.joining(",")); } if (ret.isEmpty()) { throw new IllegalArgumentException("Init raft server config error!"); @@ -374,4 +421,13 @@ public AtomicBoolean getServerStart() { public AtomicBoolean getClsStart() { return clsStart; } + + /** + * Getter method for property raftServer. + * + * @return property value of raftServer + */ + public RaftServer getRaftServer() { + return raftServer; + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/SessionNodeExchanger.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/SessionNodeExchanger.java index eeb677d8b..f2bb7364a 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/SessionNodeExchanger.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/SessionNodeExchanger.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.meta.remoting; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Request; import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -60,7 +61,7 @@ public Response request(Request request) throws RequestException { Channel channel = sessionServer.getChannel(request.getRequestUrl()); if (channel != null && channel.isConnected()) { final Object result = sessionServer.sendSync(channel, request.getRequestBody(), - metaServerConfig.getSessionNodeExchangeTimeout()); + request.getTimeout() != null ? request.getTimeout() : metaServerConfig.getSessionNodeExchangeTimeout()); response = () -> result; } else { String errorMsg = "SessionNode Exchanger get channel error! channel with url:" diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/DataConnectionHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/DataConnectionHandler.java index 58f95bee7..e788862b8 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/DataConnectionHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/DataConnectionHandler.java @@ -29,6 +29,7 @@ /** * Handle data node's connect request + * * @author shangyu.wh * @version $Id: DataConnectionHandler.java, v 0.1 2018-01-24 16:04 shangyu.wh Exp $ */ @@ -76,5 +77,4 @@ public Collection getConnections(String dataCenter) { public NodeType getNodeType() { return NodeType.DATA; } - } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/MetaConnectionHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/MetaConnectionHandler.java index 2e4699278..533198ddc 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/MetaConnectionHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/MetaConnectionHandler.java @@ -114,4 +114,5 @@ public Collection getConnections(String dataCenter) { public NodeType getNodeType() { return NodeType.META; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/SessionConnectionHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/SessionConnectionHandler.java index 49aa5e211..894b7c325 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/SessionConnectionHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/connection/SessionConnectionHandler.java @@ -75,5 +75,4 @@ public HandlerType getType() { public NodeType getNodeType() { return NodeType.SESSION; } - } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/AbstractServerHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/AbstractServerHandler.java index 5bcb55119..bea4b9dfa 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/AbstractServerHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/AbstractServerHandler.java @@ -21,9 +21,12 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.RemotingException; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; /** - * * @author shangyu.wh * @version $Id: ServerHandler.java, v 0.1 2017-11-28 18:06 shangyu.wh Exp $ */ @@ -31,6 +34,9 @@ public abstract class AbstractServerHandler implements ChannelHandler { private static final Logger LOGGER = LoggerFactory.getLogger("META-CONNECT"); + @Autowired + private ThreadPoolExecutor defaultRequestExecutor; + @Override public void connected(Channel channel) throws RemotingException { if (channel != null && channel.isConnected()) { @@ -67,4 +73,8 @@ public Class interest() { return null; } + @Override + public Executor getExecutor() { + return defaultRequestExecutor; + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/DataNodeHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/DataNodeHandler.java index 06408669f..37e8ca4f3 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/DataNodeHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/DataNodeHandler.java @@ -58,4 +58,5 @@ public Class interest() { public HandlerType getType() { return HandlerType.PROCESSER; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/FetchProvideDataRequestHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/FetchProvideDataRequestHandler.java index cef3777fd..672650b63 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/FetchProvideDataRequestHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/FetchProvideDataRequestHandler.java @@ -20,7 +20,6 @@ import com.alipay.sofa.registry.common.model.console.PersistenceData; import com.alipay.sofa.registry.common.model.metaserver.FetchProvideDataRequest; import com.alipay.sofa.registry.common.model.metaserver.ProvideData; -import com.alipay.sofa.registry.common.model.store.DataInfo; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; @@ -31,6 +30,7 @@ /** * Handle session node's query request, such as get ProvideData by dataInfoId + * * @author shangyu.wh * @version $Id: GetNodesRequestHandler.java, v 0.1 2018-03-02 15:12 shangyu.wh Exp $ */ @@ -53,10 +53,8 @@ public Object reply(Channel channel, FetchProvideDataRequest fetchProvideDataReq if (ret.getOperationStatus() == OperationStatus.SUCCESS) { PersistenceData data = (PersistenceData) ret.getEntity(); - String dataInfoId = DataInfo.toDataInfoId(data.getDataId(), data.getInstanceId(), - data.getGroup()); ProvideData provideData = new ProvideData(new ServerDataBox(data.getData()), - dataInfoId, data.getVersion()); + fetchProvideDataRequest.getDataInfoId(), data.getVersion()); DB_LOGGER.info("get ProvideData {} from DB success!", provideData); return provideData; } else if (ret.getOperationStatus() == OperationStatus.NOTFOUND) { @@ -87,4 +85,5 @@ public HandlerType getType() { public Class interest() { return FetchProvideDataRequest.class; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetChangeListRequestHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetChangeListRequestHandler.java deleted file mode 100644 index e373f9fc7..000000000 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetChangeListRequestHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alipay.sofa.registry.server.meta.remoting.handler; - -import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; -import com.alipay.sofa.registry.common.model.metaserver.GetChangeListRequest; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.remoting.Channel; -import com.alipay.sofa.registry.server.meta.registry.Registry; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * Handle other datacenter's meta node's request for get the change node list - * @author shangyu.wh - * @version $Id: GetChangeListRequestHandler.java, v 0.1 2018-02-12 16:14 shangyu.wh Exp $ - */ -public class GetChangeListRequestHandler extends AbstractServerHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger("META-CONNECT"); - - @Autowired - private Registry metaServerRegistry; - - @Override - public Object reply(Channel channel, GetChangeListRequest getChangeListRequest) { - DataCenterNodes dataCenterNodes; - try { - dataCenterNodes = metaServerRegistry.getDataCenterNodes(getChangeListRequest - .getNodeType()); - LOGGER.info("Get change Node list {} success!from {}", dataCenterNodes, - channel.getRemoteAddress()); - } catch (Exception e) { - LOGGER.error("Get change Node list error!", e); - throw new RuntimeException("Get change Node list error!", e); - } - return dataCenterNodes; - } - - @Override - public HandlerType getType() { - return HandlerType.PROCESSER; - } - - @Override - public Class interest() { - return GetChangeListRequest.class; - } -} \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetNodesRequestHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetNodesRequestHandler.java index 916d5c38e..2fb6944c7 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetNodesRequestHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/GetNodesRequestHandler.java @@ -60,4 +60,5 @@ public HandlerType getType() { public Class interest() { return GetNodesRequest.class; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/ReNewNodesRequestHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/RenewNodesRequestHandler.java similarity index 73% rename from server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/ReNewNodesRequestHandler.java rename to server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/RenewNodesRequestHandler.java index 7425ce141..d9a003260 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/ReNewNodesRequestHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/RenewNodesRequestHandler.java @@ -16,46 +16,48 @@ */ package com.alipay.sofa.registry.server.meta.remoting.handler; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; -import com.alipay.sofa.registry.common.model.metaserver.ReNewNodesRequest; +import com.alipay.sofa.registry.common.model.metaserver.RenewNodesRequest; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.server.meta.registry.Registry; -import org.springframework.beans.factory.annotation.Autowired; /** * Handle session/data node's heartbeat request * @author shangyu.wh - * @version $Id: ReNewNodesRequestHandler.java, v 0.1 2018-03-30 19:58 shangyu.wh Exp $ + * @version $Id: RenewNodesRequestHandler.java, v 0.1 2018-03-30 19:58 shangyu.wh Exp $ */ -public class ReNewNodesRequestHandler extends AbstractServerHandler { +public class RenewNodesRequestHandler extends AbstractServerHandler { - private static final Logger LOGGER = LoggerFactory.getLogger(ReNewNodesRequestHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RenewNodesRequestHandler.class); @Autowired private Registry metaServerRegistry; @Override - public Object reply(Channel channel, ReNewNodesRequest reNewNodesRequest) { - Node reNewNode = null; + public Object reply(Channel channel, RenewNodesRequest renewNodesRequest) { + Node renewNode = null; try { - reNewNode = reNewNodesRequest.getNode(); - metaServerRegistry.reNew(reNewNode, reNewNodesRequest.getDuration()); + renewNode = renewNodesRequest.getNode(); + metaServerRegistry.renew(renewNode, renewNodesRequest.getDuration()); } catch (Exception e) { - LOGGER.error("Node " + reNewNode + "reNew error!", e); - throw new RuntimeException("Node reNew error!", e); + LOGGER.error("Node " + renewNode + "renew error!", e); + throw new RuntimeException("Node renew error!", e); } return null; } @Override public Class interest() { - return ReNewNodesRequest.class; + return RenewNodesRequest.class; } @Override public HandlerType getType() { return HandlerType.PROCESSER; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/SessionNodeHandler.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/SessionNodeHandler.java index f012d76d1..613cad34f 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/SessionNodeHandler.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/remoting/handler/SessionNodeHandler.java @@ -58,4 +58,5 @@ public Class interest() { public HandlerType getType() { return HandlerType.PROCESSER; } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/RepositoryService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/RepositoryService.java index 414325dd6..396d7a9f2 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/RepositoryService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/RepositoryService.java @@ -27,11 +27,23 @@ */ public interface RepositoryService { - V put(K key, V value); + default V put(K key, V value){ + return put(key,value,System.currentTimeMillis()); + } - V remove(Object key); + default V remove(Object key){ + return remove(key,System.currentTimeMillis()); + } - V replace(K key, V value); + default V replace(K key, V value){ + return replace(key,value,System.currentTimeMillis()); + } + + V put(K key, V value,Long currentTimeMillis); + + V remove(Object key,Long currentTimeMillis); + + V replace(K key, V value,Long currentTimeMillis); @ReadOnLeader V get(Object key); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/DataRepositoryService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/DataRepositoryService.java index 716359bb3..f7c190529 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/DataRepositoryService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/DataRepositoryService.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.server.meta.repository.service; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.jraft.processor.AbstractSnapshotProcess; import com.alipay.sofa.registry.jraft.processor.SnapshotProcess; @@ -26,15 +36,6 @@ import com.alipay.sofa.registry.server.meta.repository.RepositoryService; import com.alipay.sofa.registry.server.meta.store.RenewDecorate; import com.alipay.sofa.registry.store.api.annotation.RaftService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @@ -84,7 +85,8 @@ public SnapshotProcess copy() { } @Override - public RenewDecorate put(String ipAddress, RenewDecorate dataNode) { + public RenewDecorate put(String ipAddress, RenewDecorate dataNode, + Long currentTimeMillis) { write.lock(); try { @@ -93,14 +95,14 @@ public RenewDecorate put(String ipAddress, RenewDecorate dat NodeRepository dataNodeRepository = registry.get(dataCenter); if (dataNodeRepository == null) { NodeRepository nodeRepository = new NodeRepository<>(dataCenter, - new ConcurrentHashMap<>(), System.currentTimeMillis()); + new ConcurrentHashMap<>(), currentTimeMillis); dataNodeRepository = registry.put(dataCenter, nodeRepository); if (dataNodeRepository == null) { dataNodeRepository = nodeRepository; } } - dataNodeRepository.setVersion(System.currentTimeMillis()); + dataNodeRepository.setVersion(currentTimeMillis); Map> dataNodes = dataNodeRepository .getNodeMap(); @@ -122,7 +124,7 @@ public RenewDecorate put(String ipAddress, RenewDecorate dat } @Override - public RenewDecorate remove(Object key) { + public RenewDecorate remove(Object key, Long currentTimeMillis) { write.lock(); try { @@ -141,7 +143,7 @@ public RenewDecorate remove(Object key) { return null; } - dataNodeRepository.setVersion(System.currentTimeMillis()); + dataNodeRepository.setVersion(currentTimeMillis); return oldRenewDecorate; } } @@ -155,7 +157,8 @@ public RenewDecorate remove(Object key) { } @Override - public RenewDecorate replace(String ipAddress, RenewDecorate dataNode) { + public RenewDecorate replace(String ipAddress, RenewDecorate dataNode, + Long currentTimeMillis) { write.lock(); try { @@ -170,7 +173,7 @@ public RenewDecorate replace(String ipAddress, RenewDecorate RenewDecorate oldRenewDecorate = dataNodes.get(ipAddress); if (oldRenewDecorate != null && oldRenewDecorate.getRenewal() != null) { oldRenewDecorate.setRenewal(dataNode.getRenewal()); - oldRenewDecorate.reNew(); + oldRenewDecorate.renew(); } else { LOGGER.error("Data node with ipAddress {} has not existed!", ipAddress); throw new RuntimeException(String.format( diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/MetaRepositoryService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/MetaRepositoryService.java index 06bb4a26d..fa88ac867 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/MetaRepositoryService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/MetaRepositoryService.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.server.meta.repository.service; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.jraft.processor.AbstractSnapshotProcess; import com.alipay.sofa.registry.jraft.processor.SnapshotProcess; @@ -26,15 +36,6 @@ import com.alipay.sofa.registry.server.meta.repository.RepositoryService; import com.alipay.sofa.registry.server.meta.store.RenewDecorate; import com.alipay.sofa.registry.store.api.annotation.RaftService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @@ -84,7 +85,8 @@ public SnapshotProcess copy() { } @Override - public RenewDecorate put(String ipAddress, RenewDecorate metaNode) { + public RenewDecorate put(String ipAddress, RenewDecorate metaNode, + Long currentTimeMillis) { write.lock(); try { String dataCenter = metaNode.getRenewal().getDataCenter(); @@ -92,14 +94,14 @@ public RenewDecorate put(String ipAddress, RenewDecorate met NodeRepository metaNodeRepository = registry.get(dataCenter); if (metaNodeRepository == null) { NodeRepository nodeRepository = new NodeRepository<>(dataCenter, - new ConcurrentHashMap<>(), System.currentTimeMillis()); + new ConcurrentHashMap<>(), currentTimeMillis); metaNodeRepository = registry.put(dataCenter, nodeRepository); if (metaNodeRepository == null) { metaNodeRepository = nodeRepository; } } - metaNodeRepository.setVersion(System.currentTimeMillis()); + metaNodeRepository.setVersion(currentTimeMillis); Map> metaNodes = metaNodeRepository .getNodeMap(); @@ -121,7 +123,7 @@ public RenewDecorate put(String ipAddress, RenewDecorate met } @Override - public RenewDecorate remove(Object key) { + public RenewDecorate remove(Object key, Long currentTimeMillis) { write.lock(); try { String ipAddress = (String) key; @@ -139,7 +141,7 @@ public RenewDecorate remove(Object key) { return null; } - metaNodeRepository.setVersion(System.currentTimeMillis()); + metaNodeRepository.setVersion(currentTimeMillis); return oldRenewDecorate; } } @@ -153,7 +155,8 @@ public RenewDecorate remove(Object key) { } @Override - public RenewDecorate replace(String ipAddress, RenewDecorate metaNode) { + public RenewDecorate replace(String ipAddress, RenewDecorate metaNode, + Long currentTimeMillis) { write.lock(); try { String dataCenter = metaNode.getRenewal().getDataCenter(); @@ -168,9 +171,9 @@ public RenewDecorate replace(String ipAddress, RenewDecorate RenewDecorate oldRenewDecorate = dataNodes.get(ipAddress); if (oldRenewDecorate != null && oldRenewDecorate.getRenewal() != null) { oldRenewDecorate.setRenewal(metaNode.getRenewal()); - oldRenewDecorate.reNew(); + oldRenewDecorate.renew(); - metaNodeRepository.setVersion(System.currentTimeMillis()); + metaNodeRepository.setVersion(currentTimeMillis); } else { LOGGER.error("Meta node with ipAddress {} has not existed!", ipAddress); throw new RuntimeException(String.format( @@ -347,4 +350,13 @@ public Set getSnapshotFileNames() { snapShotFileNames.add(this.getClass().getSimpleName()); return snapShotFileNames; } + + /** + * Setter method for property nodeConfig. + * + * @param nodeConfig value to be assigned to property nodeConfig + */ + public void setNodeConfig(NodeConfig nodeConfig) { + this.nodeConfig = nodeConfig; + } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/SessionRepositoryService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/SessionRepositoryService.java index 4a4f4d10f..1b2b3daf2 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/SessionRepositoryService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/repository/service/SessionRepositoryService.java @@ -16,6 +16,14 @@ */ package com.alipay.sofa.registry.server.meta.repository.service; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.metaserver.SessionNode; import com.alipay.sofa.registry.jraft.processor.AbstractSnapshotProcess; import com.alipay.sofa.registry.jraft.processor.SnapshotProcess; @@ -26,13 +34,6 @@ import com.alipay.sofa.registry.server.meta.repository.RepositoryService; import com.alipay.sofa.registry.server.meta.store.RenewDecorate; import com.alipay.sofa.registry.store.api.annotation.RaftService; -import org.springframework.beans.factory.annotation.Autowired; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * @@ -77,7 +78,8 @@ public SnapshotProcess copy() { } @Override - public RenewDecorate put(String ipAddress, RenewDecorate sessionNode) { + public RenewDecorate put(String ipAddress, RenewDecorate sessionNode, + Long currentTimeMillis) { try { RenewDecorate oldRenewDecorate = registry.get(ipAddress); if (oldRenewDecorate != null && oldRenewDecorate.getRenewal() != null) { @@ -93,7 +95,7 @@ public RenewDecorate put(String ipAddress, RenewDecorate remove(Object key) { + public RenewDecorate remove(Object key, Long currentTimeMillis) { try { String ipAddress = (String) key; RenewDecorate oldRenewDecorate = registry.remove(ipAddress); @@ -111,11 +113,12 @@ public RenewDecorate remove(Object key) { @Override public RenewDecorate replace(String ipAddress, - RenewDecorate sessionNode) { + RenewDecorate sessionNode, + Long currentTimeMillis) { RenewDecorate oldRenewDecorate = registry.get(ipAddress); if (oldRenewDecorate != null && oldRenewDecorate.getRenewal() != null) { oldRenewDecorate.setRenewal(sessionNode.getRenewal()); - oldRenewDecorate.reNew(); + oldRenewDecorate.renew(); } else { LOGGER.error("Session node with ipAddress {} has not existed!", ipAddress); throw new RuntimeException(String.format( diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/BlacklistDataResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/BlacklistDataResource.java new file mode 100644 index 000000000..f11320812 --- /dev/null +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/BlacklistDataResource.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.resource; + +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.console.PersistenceData; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.DataOperator; +import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.core.model.Result; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.store.api.DBService; +import com.alipay.sofa.registry.store.api.annotation.RaftReference; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; + +/** + * + * @author shangyu.wh + * @version $Id: StopPushDataResource.java, v 0.1 2018-07-25 11:40 shangyu.wh Exp $ + */ +@Path("blacklist") +public class BlacklistDataResource { + + private static final Logger DB_LOGGER = LoggerFactory.getLogger(BlacklistDataResource.class, + "[DBService]"); + + private static final Logger TASK_LOGGER = LoggerFactory.getLogger(BlacklistDataResource.class, + "[Task]"); + + @RaftReference + private DBService persistenceDataDBService; + + @Autowired + private TaskListenerManager taskListenerManager; + + /** + * update blacklist + * e.g. curl -d '{"FORBIDDEN_PUB":{"IP_FULL":["1.1.1.1","10.15.233.150"]},"FORBIDDEN_SUB_BY_PREFIX":{"IP_FULL":["1.1.1.1"]}}' -H "Content-Type: application/json" -X POST http://localhost:9615/blacklist/update + */ + @POST + @Path("update") + @Produces(MediaType.APPLICATION_JSON) + public Result blacklistPush(String config) { + PersistenceData persistenceData = createDataInfo(); + persistenceData.setData(config); + try { + boolean ret = persistenceDataDBService.update(ValueConstants.BLACK_LIST_DATA_ID, + persistenceData); + DB_LOGGER.info("Success update blacklist to DB result {}!", ret); + } catch (Exception e) { + DB_LOGGER.error("Error update blacklist to DB!", e); + throw new RuntimeException("Update blacklist to error!"); + } + + fireDataChangeNotify(persistenceData.getVersion(), ValueConstants.BLACK_LIST_DATA_ID, + DataOperator.UPDATE); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + private PersistenceData createDataInfo() { + DataInfo dataInfo = DataInfo.valueOf(ValueConstants.BLACK_LIST_DATA_ID); + PersistenceData persistenceData = new PersistenceData(); + persistenceData.setDataId(dataInfo.getDataId()); + persistenceData.setGroup(dataInfo.getDataType()); + persistenceData.setInstanceId(dataInfo.getInstanceId()); + persistenceData.setVersion(System.currentTimeMillis()); + return persistenceData; + } + + private void fireDataChangeNotify(Long version, String dataInfoId, DataOperator dataOperator) { + + NotifyProvideDataChange notifyProvideDataChange = new NotifyProvideDataChange(dataInfoId, + version, dataOperator); + + TaskEvent taskEvent = new TaskEvent(notifyProvideDataChange, + TaskType.PERSISTENCE_DATA_CHANGE_NOTIFY_TASK); + TASK_LOGGER.info("send PERSISTENCE_DATA_CHANGE_NOTIFY_TASK notifyProvideDataChange:" + + notifyProvideDataChange); + taskListenerManager.sendTaskEvent(taskEvent); + } + +} \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/HealthResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/HealthResource.java index 8e6970390..616445b64 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/HealthResource.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/HealthResource.java @@ -18,14 +18,21 @@ import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.jraft.bootstrap.ServiceStateMachine; +import com.alipay.sofa.registry.metrics.ReporterUtils; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerBootstrap; import com.alipay.sofa.registry.server.meta.remoting.RaftExchanger; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; import org.springframework.beans.factory.annotation.Autowired; +import javax.annotation.PostConstruct; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.Response.Status; /** * @@ -41,11 +48,28 @@ public class HealthResource { @Autowired private RaftExchanger raftExchanger; + @PostConstruct + public void init() { + MetricRegistry metrics = new MetricRegistry(); + metrics.register("healthCheck", (Gauge) () -> getHealthCheckResult()); + ReporterUtils.startSlf4jReporter(60, metrics); + } + @GET @Path("check") @Produces(MediaType.APPLICATION_JSON) - public CommonResponse checkHealth() { + public Response checkHealth() { + ResponseBuilder builder = Response.status(Response.Status.OK); + CommonResponse response = getHealthCheckResult(); + builder.entity(response); + if (!response.isSuccess()) { + builder.status(Status.INTERNAL_SERVER_ERROR); + } + return builder.build(); + } + + private CommonResponse getHealthCheckResult() { CommonResponse response; StringBuilder sb = new StringBuilder("MetaServerBoot "); @@ -96,7 +120,6 @@ public CommonResponse checkHealth() { } else { response = CommonResponse.buildFailedResponse(sb.toString()); } - return response; } } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/MetaDigestResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/MetaDigestResource.java index 538614f15..42211be7c 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/MetaDigestResource.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/MetaDigestResource.java @@ -16,26 +16,32 @@ */ package com.alipay.sofa.registry.server.meta.resource; +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.console.PersistenceData; import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.metrics.ReporterUtils; import com.alipay.sofa.registry.server.meta.registry.Registry; import com.alipay.sofa.registry.store.api.DBResponse; import com.alipay.sofa.registry.store.api.DBService; import com.alipay.sofa.registry.store.api.OperationStatus; import com.alipay.sofa.registry.store.api.annotation.RaftReference; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import java.util.HashMap; -import java.util.Map; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; /** * @@ -57,6 +63,16 @@ public class MetaDigestResource { @RaftReference private DBService persistenceDataDBService; + @PostConstruct + public void init() { + MetricRegistry metrics = new MetricRegistry(); + metrics.register("metaNodeList", (Gauge) () -> getRegisterNodeByType("meta")); + metrics.register("dataNodeList", (Gauge) () -> getRegisterNodeByType("data")); + metrics.register("sessionNodeList", (Gauge) () -> getRegisterNodeByType("session")); + metrics.register("pushSwitch", (Gauge) () -> getPushSwitch()); + ReporterUtils.startSlf4jReporter(60, metrics); + } + @GET @Path("{type}/node/query") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/RenewSwitchResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/RenewSwitchResource.java new file mode 100644 index 000000000..f4f63b397 --- /dev/null +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/RenewSwitchResource.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.resource; + +import java.util.HashMap; +import java.util.Map; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.Node.NodeType; +import com.alipay.sofa.registry.common.model.console.PersistenceData; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.DataOperator; +import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.core.model.Result; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.store.api.DBResponse; +import com.alipay.sofa.registry.store.api.DBService; +import com.alipay.sofa.registry.store.api.annotation.RaftReference; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import com.google.common.collect.Sets; + +/** + * + * @author shangyu.wh + * @version $Id: StopPushDataResource.java, v 0.1 2018-07-25 11:40 shangyu.wh Exp $ + */ +@Path("renewSwitch") +public class RenewSwitchResource { + + private static final Logger DB_LOGGER = LoggerFactory.getLogger(RenewSwitchResource.class, + "[DBService]"); + + private static final Logger TASK_LOGGER = LoggerFactory.getLogger(RenewSwitchResource.class, + "[Task]"); + + @RaftReference + private DBService persistenceDataDBService; + + @Autowired + private TaskListenerManager taskListenerManager; + + /** + * get + */ + @GET + @Path("get") + @Produces(MediaType.APPLICATION_JSON) + public Map getPushSwitch() throws Exception { + Map resultMap = new HashMap<>(1); + DBResponse enableDataRenewSnapshot = persistenceDataDBService + .get(ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT); + DBResponse enableDataDatumExpire = persistenceDataDBService + .get(ValueConstants.ENABLE_DATA_DATUM_EXPIRE); + + resultMap + .put( + "enableDataRenewSnapshot", + enableDataRenewSnapshot != null && enableDataRenewSnapshot.getEntity() != null ? ((PersistenceData) enableDataRenewSnapshot + .getEntity()).getData() : "null"); + resultMap + .put( + "enableDataDatumExpire", + enableDataDatumExpire != null && enableDataDatumExpire.getEntity() != null ? ((PersistenceData) enableDataDatumExpire + .getEntity()).getData() : "null"); + + return resultMap; + } + + /** + * enable both + */ + @GET + @Path("enable") + @Produces(MediaType.APPLICATION_JSON) + public Result enableRenew() { + invokeSession("true"); + invokeData("true"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + /** + * disable both + */ + @GET + @Path("disable") + @Produces(MediaType.APPLICATION_JSON) + public Result disableRenew() { + invokeSession("false"); + invokeData("false"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + /** + * enable session + */ + @GET + @Path("session/enable") + @Produces(MediaType.APPLICATION_JSON) + public Result enableSessionRenew() { + invokeSession("true"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + /** + * disable session + */ + @GET + @Path("session/disable") + @Produces(MediaType.APPLICATION_JSON) + public Result disableSessionRenew() { + invokeSession("false"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + /** + * enable data + */ + @GET + @Path("data/enable") + @Produces(MediaType.APPLICATION_JSON) + public Result enableDataRenew() { + invokeData("true"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + /** + * disable data + */ + @GET + @Path("data/disable") + @Produces(MediaType.APPLICATION_JSON) + public Result disableDataRenew() { + invokeData("false"); + + Result result = new Result(); + result.setSuccess(true); + return result; + } + + private void invokeSession(String data) { + String msg = "put ENABLE_DATA_RENEW_SNAPSHOT to DB"; + PersistenceData persistenceData = createPersistenceData(ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT); + persistenceData.setData(data); + try { + boolean ret = persistenceDataDBService.put(ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT, + persistenceData); + DB_LOGGER.info(String.format("%s result %s", msg, ret)); + } catch (Exception e) { + DB_LOGGER.error(String.format("Error %s: %s", msg, e.getMessage())); + throw new RuntimeException(String.format("Error %s: %s", msg, e.getMessage()), e); + } + + fireDataChangeNotify(NodeType.SESSION, persistenceData.getVersion(), + ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT, DataOperator.UPDATE); + } + + private void invokeData(String data) { + String msg = "put ENABLE_DATA_DATUM_EXPIRE to DB"; + PersistenceData persistenceData = createPersistenceData(ValueConstants.ENABLE_DATA_DATUM_EXPIRE); + persistenceData.setData(data); + try { + boolean ret = persistenceDataDBService.put(ValueConstants.ENABLE_DATA_DATUM_EXPIRE, + persistenceData); + DB_LOGGER.info(String.format("%s result %s", msg, ret)); + } catch (Exception e) { + DB_LOGGER.error(String.format("Error %s: %s", msg, e.getMessage())); + throw new RuntimeException(String.format("Error %s: %s", msg, e.getMessage()), e); + } + + fireDataChangeNotify(NodeType.DATA, persistenceData.getVersion(), + ValueConstants.ENABLE_DATA_DATUM_EXPIRE, DataOperator.UPDATE); + } + + private PersistenceData createPersistenceData(String dataId) { + DataInfo dataInfo = DataInfo.valueOf(dataId); + PersistenceData persistenceData = new PersistenceData(); + persistenceData.setDataId(dataInfo.getDataId()); + persistenceData.setGroup(dataInfo.getDataType()); + persistenceData.setInstanceId(dataInfo.getInstanceId()); + persistenceData.setVersion(System.currentTimeMillis()); + return persistenceData; + } + + private void fireDataChangeNotify(NodeType nodeType, Long version, String dataInfoId, + DataOperator dataOperator) { + NotifyProvideDataChange notifyProvideDataChange = new NotifyProvideDataChange(dataInfoId, + version, dataOperator); + notifyProvideDataChange.setNodeTypes(Sets.newHashSet(nodeType)); + + TaskEvent taskEvent = new TaskEvent(notifyProvideDataChange, + TaskType.PERSISTENCE_DATA_CHANGE_NOTIFY_TASK); + TASK_LOGGER.info("send PERSISTENCE_DATA_CHANGE_NOTIFY_TASK notifyProvideDataChange:" + + notifyProvideDataChange); + taskListenerManager.sendTaskEvent(taskEvent); + } + +} \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/SessionLoadbalanceResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/SessionLoadbalanceResource.java new file mode 100644 index 000000000..0f985ff27 --- /dev/null +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/SessionLoadbalanceResource.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.resource; + +import com.alipay.sofa.registry.server.meta.registry.Registry; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; + +/** + * @author xiangxu + * @version : SessionLoadbalanceSwitchResource.java, v 0.1 2020年06月02日 2:51 下午 xiangxu Exp $ + */ + +@Path("sessionLoadbalance") +public class SessionLoadbalanceResource { + @Autowired + private Registry metaServerRegistry; + + @POST + @Path("run") + @Produces(MediaType.APPLICATION_JSON) + public Object run(@FormParam("maxDisconnect") @DefaultValue("0") String maxDisconnect) { + return metaServerRegistry.sessionLoadbalance(Integer.parseInt(maxDisconnect)); + } +} \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/StopPushDataResource.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/StopPushDataResource.java index 40df5eec1..6083d7ff6 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/StopPushDataResource.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/resource/StopPushDataResource.java @@ -16,6 +16,16 @@ */ package com.alipay.sofa.registry.server.meta.resource; +import java.util.Set; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.console.PersistenceData; import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.metaserver.DataOperator; @@ -29,12 +39,7 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; +import com.google.common.collect.Sets; /** * @@ -122,7 +127,7 @@ private PersistenceData createPushDataInfo() { private void fireDataChangeNotify(Long version, String dataInfoId, DataOperator dataOperator) { NotifyProvideDataChange notifyProvideDataChange = new NotifyProvideDataChange(dataInfoId, - version, dataOperator); + version, dataOperator, getNodeTypes()); TaskEvent taskEvent = new TaskEvent(notifyProvideDataChange, TaskType.PERSISTENCE_DATA_CHANGE_NOTIFY_TASK); @@ -131,4 +136,8 @@ private void fireDataChangeNotify(Long version, String dataInfoId, DataOperator taskListenerManager.sendTaskEvent(taskEvent); } + protected Set getNodeTypes() { + return Sets.newHashSet(NodeType.SESSION); + } + } \ No newline at end of file diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/DataStoreService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/DataStoreService.java index ff324ff19..825bc80c4 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/DataStoreService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/DataStoreService.java @@ -16,6 +16,21 @@ */ package com.alipay.sofa.registry.server.meta.store; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.DataNode; @@ -36,20 +51,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; /** * @@ -85,8 +86,6 @@ public class DataStoreService implements StoreService { private AtomicLong localDataCenterInitVersion = new AtomicLong( -1L); - private static final long COMPARE_TIME_COST = 1000L; - @Override public NodeType getNodeType() { return NodeType.DATA; @@ -103,14 +102,13 @@ public NodeChangeResult addNode(DataNode dataNode) { String ipAddress = dataNode.getNodeUrl().getIpAddress(); - long startAll = System.currentTimeMillis(); write.lock(); try { dataRepositoryService.put(ipAddress, new RenewDecorate(dataNode, RenewDecorate.DEFAULT_DURATION_SECS)); - reNew(dataNode, 30); + renew(dataNode, 30); nodeChangeResult = getNodeChangeResult(); @@ -119,10 +117,6 @@ public NodeChangeResult addNode(DataNode dataNode) { } finally { write.unlock(); } - long cost = System.currentTimeMillis() - startAll; - if (cost >= COMPARE_TIME_COST) { - LOGGER.info("dataRepositoryService.addNode cost:{} ", cost); - } return nodeChangeResult; } @@ -170,15 +164,14 @@ public void removeNodes(Collection nodes) { } @Override - public void reNew(DataNode dataNode, int duration) { + public void renew(DataNode dataNode, int duration) { - long startAll = System.currentTimeMillis(); write.lock(); try { String ipAddress = dataNode.getNodeUrl().getIpAddress(); - RenewDecorate reNewer = dataRepositoryService.get(ipAddress); + RenewDecorate renewer = dataRepositoryService.get(ipAddress); - if (reNewer == null) { + if (renewer == null) { LOGGER.warn("Renew Data node with ipAddress:" + ipAddress + " has not existed!It will be registered again!"); addNode(dataNode); @@ -191,10 +184,6 @@ public void reNew(DataNode dataNode, int duration) { } } - long cost = System.currentTimeMillis() - startAll; - if (cost >= COMPARE_TIME_COST) { - LOGGER.info("dataRepositoryService.renew.all cost:{} ", cost); - } } finally { write.unlock(); } @@ -206,7 +195,7 @@ public void reNew(DataNode dataNode, int duration) { */ @Override public Collection getExpired() { - Collection reNewerList = new ArrayList<>(); + Collection renewerList = new ArrayList<>(); read.lock(); try { Map> dataMap = dataRepositoryService.getAllData(); @@ -216,7 +205,7 @@ public Collection getExpired() { String dataCenter = dataNode.getRenewal().getDataCenter(); if (dataCenter.equals(nodeConfig.getLocalDataCenter())) { if (dataNode.isExpired()) { - reNewerList.add(dataNode.getRenewal()); + renewerList.add(dataNode.getRenewal()); } } }); @@ -224,7 +213,7 @@ public Collection getExpired() { } finally { read.unlock(); } - return reNewerList; + return renewerList; } @Override @@ -444,6 +433,7 @@ private Map> getRunTime() { } } + //TODO move this code to enterprise version @Override public void getOtherDataCenterNodeAndUpdate() { @@ -455,38 +445,45 @@ public void getOtherDataCenterNodeAndUpdate() { if (metaMap != null && metaMap.size() > 0) { for (String dataCenter : metaMap.keySet()) { //get other dataCenter dataNodes - if (!nodeConfig.getLocalDataCenter().equals(dataCenter)) { - GetChangeListRequest getChangeListRequest = new GetChangeListRequest( - NodeType.DATA, dataCenter); - //trigger fetch dataCenter data list change - DataCenterNodes getDataCenterNodes = metaNodeService - .getDataCenterNodes(getChangeListRequest); - if (getDataCenterNodes != null) { + try { + if (!nodeConfig.getLocalDataCenter().equals(dataCenter)) { + GetChangeListRequest getChangeListRequest = new GetChangeListRequest( + NodeType.DATA, dataCenter); + //trigger fetch dataCenter data list change + DataCenterNodes getDataCenterNodes = metaNodeService + .getDataCenterNodes(getChangeListRequest); + LOGGER.info("GetOtherDataCenterNode from DataCenter({}): {}", dataCenter, + getDataCenterNodes); String dataCenterGet = getDataCenterNodes.getDataCenterId(); Long version = getDataCenterNodes.getVersion(); if (version == null) { - LOGGER.error("Request message version cant not be null!"); - return; + LOGGER + .error( + "getOtherDataCenterNodeAndUpdate from DataCenter({}), data list version is null", + dataCenter); + continue; } //check for scheduler get other dataCenter data node boolean result = dataRepositoryService.checkVersion(dataCenterGet, version); if (!result) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("DataCenter {} data list version {} has not updated!", + LOGGER + .warn( + "getOtherDataCenterNodeAndUpdate from DataCenter({}), data list version {} has not updated", dataCenter, version); - } - return; + continue; } updateOtherDataCenterNodes(getDataCenterNodes); - } else { - LOGGER.error("Get DataCenter data nodes change error!null"); - throw new RuntimeException("Get null DataCenter data nodes change!"); } + } catch (Throwable e) { + LOGGER.error(String.format( + "getOtherDataCenterNodeAndUpdate from DataCenter(%s) error: %s", + dataCenter, e.getMessage()), e); } } } } + //TODO move this to enterprise version @Override public void updateOtherDataCenterNodes(DataCenterNodes dataCenterNodes) { write.lock(); diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/MetaStoreService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/MetaStoreService.java index f50ce3b80..457469f39 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/MetaStoreService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/MetaStoreService.java @@ -16,6 +16,20 @@ */ package com.alipay.sofa.registry.server.meta.store; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import javax.ws.rs.NotSupportedException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.GetChangeListRequest; @@ -31,18 +45,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.ws.rs.NotSupportedException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @@ -167,7 +169,7 @@ public void confirmNodeStatus(String connectId, String ip) { } @Override - public void reNew(MetaNode node, int duration) { + public void renew(MetaNode node, int duration) { } @Override @@ -216,6 +218,7 @@ public NodeChangeResult getNodeChangeResult() { return nodeChangeResult; } + //TODO move this code to enterprise version @Override public void getOtherDataCenterNodeAndUpdate() { @@ -227,36 +230,48 @@ public void getOtherDataCenterNodeAndUpdate() { if (metaMap != null && metaMap.size() > 0) { for (String dataCenter : metaMap.keySet()) { //get other dataCenter meta - if (!nodeConfig.getLocalDataCenter().equals(dataCenter)) { - GetChangeListRequest getChangeListRequest = new GetChangeListRequest( - NodeType.META, dataCenter); - //trigger fetch dataCenter meta list change - DataCenterNodes getDataCenterNodes = metaNodeService - .getDataCenterNodes(getChangeListRequest); - if (getDataCenterNodes != null) { + try { + if (!nodeConfig.getLocalDataCenter().equals(dataCenter)) { + + GetChangeListRequest getChangeListRequest = new GetChangeListRequest( + NodeType.META, dataCenter); + //trigger fetch dataCenter meta list change + DataCenterNodes getDataCenterNodes = metaNodeService + .getDataCenterNodes(getChangeListRequest); String dataCenterGet = getDataCenterNodes.getDataCenterId(); + + LOGGER.info("GetOtherDataCenterNode from DataCenter({}): {}", dataCenter, + getDataCenterNodes); + Long version = getDataCenterNodes.getVersion(); if (version == null) { - LOGGER.error("Request message meta version cant not be null!"); - return; + LOGGER + .error( + "getOtherDataCenterNodeAndUpdate from DataCenter({}), meta list version is null", + dataCenter); + continue; } //check for scheduler get other dataCenter meta node boolean result = metaRepositoryService.checkVersion(dataCenterGet, version); if (!result) { - LOGGER.debug("DataCenter {} meta list version {} has not updated!", - dataCenter, version); - return; + LOGGER + .warn( + "getOtherDataCenterNodeAndUpdate from DataCenter({}), meta list version {} has not updated", + dataCenter, version); + continue; } updateOtherDataCenterNodes(getDataCenterNodes); - } else { - LOGGER.error("Get DataCenter meta nodes change error!null"); - throw new RuntimeException("Get null DataCenter meta nodes change!"); } + } catch (Throwable e) { + LOGGER.error(String.format( + "getOtherDataCenterNodeAndUpdate from DataCenter(%s) error: %s", + dataCenter, e.getMessage()), e); } } } } + //TODO move this to enterprise version @Override public void updateOtherDataCenterNodes(DataCenterNodes dataCenterNodes) { write.lock(); @@ -335,9 +350,8 @@ private void firePushDataListTask(NodeChangeResult nodeChangeResult, String node taskEvent.setAttribute(Constant.PUSH_TARGET_TYPE, NodeType.DATA); taskEvent.setAttribute(Constant.PUSH_TARGET_OPERATOR_TYPE, nodeOperate); - LOGGER.info("send " + taskEvent.getTaskType() + " NodeType:" + NodeType.DATA + " Operator:" - + nodeOperate); - + LOGGER.info("send {} NodeType:{} Operator:{}", taskEvent.getTaskType(), NodeType.DATA, + nodeOperate); taskListenerManager.sendTaskEvent(taskEvent); } @@ -347,8 +361,8 @@ private void firePushSessionListTask(NodeChangeResult nodeChangeResult, String n TaskEvent taskEvent = new TaskEvent(nodeChangeResult, TaskType.DATA_NODE_CHANGE_PUSH_TASK); taskEvent.setAttribute(Constant.PUSH_TARGET_TYPE, NodeType.SESSION); taskEvent.setAttribute(Constant.PUSH_TARGET_OPERATOR_TYPE, nodeOperate); - LOGGER.info("send " + taskEvent.getTaskType() + " NodeType:" + NodeType.SESSION - + " Operator:" + nodeOperate); + LOGGER.info("send {} NodeType:{} Operator:{}", taskEvent.getTaskType(), NodeType.SESSION, + nodeOperate); taskListenerManager.sendTaskEvent(taskEvent); } diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/RenewDecorate.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/RenewDecorate.java index e642ba557..2127a2c1b 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/RenewDecorate.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/RenewDecorate.java @@ -21,7 +21,7 @@ /** * heartbeat info for node * @author shangyu.wh - * @version $Id: ReNewer.java, v 0.1 2018-01-16 17:10 shangyu.wh Exp $ + * @version $Id: Renewer.java, v 0.1 2018-01-16 17:10 shangyu.wh Exp $ */ public class RenewDecorate implements Serializable { @@ -66,7 +66,7 @@ public boolean isExpired() { /** * refresh lastUpdateTimestamp */ - public void reNew() { + public void renew() { lastUpdateTimestamp = System.currentTimeMillis() + duration; } @@ -74,7 +74,7 @@ public void reNew() { * refresh lastUpdateTimestamp by durationSECS * @param durationSECS */ - public void reNew(long durationSECS) { + public void renew(long durationSECS) { lastUpdateTimestamp = System.currentTimeMillis() + durationSECS * 1000; } diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/SessionStoreService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/SessionStoreService.java index 5123a9f43..213724087 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/SessionStoreService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/SessionStoreService.java @@ -16,12 +16,28 @@ */ package com.alipay.sofa.registry.server.meta.store; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +import javax.ws.rs.NotSupportedException; + +import com.alipay.sofa.registry.common.model.metaserver.*; +import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; +import com.alipay.sofa.registry.server.meta.bootstrap.ServiceFactory; +import com.alipay.sofa.registry.server.meta.node.SessionNodeService; +import com.alipay.sofa.registry.store.api.DBService; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; -import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; -import com.alipay.sofa.registry.common.model.metaserver.DataOperator; -import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import com.alipay.sofa.registry.common.model.metaserver.SessionNode; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; @@ -34,22 +50,8 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.ws.rs.NotSupportedException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; /** - * * @author shangyu.wh * @version $Id: SessionStoreService.java, v 0.1 2018-01-12 14:14 shangyu.wh Exp $ */ @@ -73,6 +75,8 @@ public class SessionStoreService implements StoreService { private StoreService dataStoreService; @Autowired private NodeConfig nodeConfig; + @Autowired + private MetaServerConfig metaServerConfig; @RaftReference(uniqueId = "sessionServer") private RepositoryService> sessionRepositoryService; @@ -83,6 +87,9 @@ public class SessionStoreService implements StoreService { @RaftReference(uniqueId = "sessionServer") private NodeConfirmStatusService sessionConfirmStatusService; + @RaftReference + private DBService dbService; + @Override public NodeType getNodeType() { return NodeType.SESSION; @@ -106,7 +113,7 @@ public NodeChangeResult addNode(SessionNode sessionNode) { sessionVersionRepositoryService.checkAndUpdateVersions(nodeConfig.getLocalDataCenter(), System.currentTimeMillis()); - reNew(sessionNode, 30); + renew(sessionNode, 30); sessionConfirmStatusService.putConfirmNode(sessionNode, DataOperator.ADD); @@ -173,15 +180,15 @@ public void removeNodes(Collection nodes) { } @Override - public void reNew(SessionNode sessionNode, int duration) { + public void renew(SessionNode sessionNode, int duration) { write.lock(); try { String ipAddress = sessionNode.getNodeUrl().getIpAddress(); - RenewDecorate reNewer = sessionRepositoryService.get(ipAddress); + RenewDecorate renewer = sessionRepositoryService.get(ipAddress); - if (reNewer == null) { - LOGGER.warn("ReNew session node with ipAddress:" + ipAddress + if (renewer == null) { + LOGGER.warn("Renew session node with ipAddress:" + ipAddress + " has not existed!It will be registered again!"); addNode(sessionNode); } else { @@ -200,19 +207,19 @@ public void reNew(SessionNode sessionNode, int duration) { @Override public Collection getExpired() { - Collection reNewerList = new ArrayList<>(); + Collection renewerList = new ArrayList<>(); read.lock(); try { Map> map = sessionRepositoryService.getAllData(); map.forEach((key, value) -> { if (value.isExpired()) { - reNewerList.add(value.getRenewal()); + renewerList.add(value.getRenewal()); } }); } finally { read.unlock(); } - return reNewerList; + return renewerList; } @Override @@ -321,7 +328,8 @@ private Set getRemoveIp(Set waitNotifyNodes) { Map> map = nodeChangeResult.getNodes(); Map addNodes = map.get(nodeConfig.getLocalDataCenter()); if (addNodes != null && !addNodes.isEmpty()) { - return waitNotifyNodes.stream().filter(ip -> !addNodes.keySet().contains(ip)).collect(Collectors.toSet()); + return waitNotifyNodes.stream().filter(ip -> !addNodes.keySet().contains(ip)) + .collect(Collectors.toSet()); } return new HashSet<>(); } @@ -359,7 +367,8 @@ public NodeChangeResult getNodeChangeResult() { nodes.put(localDataCenter, tmpMap); nodeChangeResult.setNodes(nodes); - nodeChangeResult.setVersion(sessionVersionRepositoryService.getVersion(nodeConfig.getLocalDataCenter())); + nodeChangeResult.setVersion( + sessionVersionRepositoryService.getVersion(nodeConfig.getLocalDataCenter())); } finally { read.unlock(); } @@ -385,10 +394,50 @@ public void updateOtherDataCenterNodes(DataCenterNodes dataCenterNodes) { throw new NotSupportedException("Node type SESSION not support function"); } + private Map zoneMaxConnections(Map nodes, int maxDisconnect) { + double thresholdRatio = metaServerConfig.getSessionLoadbalanceThresholdRatio(); + Map maxConnectionsMap = new HashMap<>(); + if (nodes.size() == 0) { + return maxConnectionsMap; + } + double avg = nodes.values().stream().mapToInt(LoadbalanceMetrics::getConnectionCount) + .average().getAsDouble(); + for (String ipAddress : nodes.keySet()) { + LoadbalanceMetrics m = nodes.get(ipAddress); + int maxConnections = (int) Math + .ceil(Math.max(avg * thresholdRatio, m.getConnectionCount() - maxDisconnect)); + maxConnectionsMap.put(ipAddress, maxConnections); + } + return maxConnectionsMap; + } + + public Map>> sessionLoadbalance(int maxDisconnect) { + Map>> result = new HashMap<>(); + Map> sourceConnections = new HashMap<>(); + + SessionNodeService sessionNodeService = (SessionNodeService) ServiceFactory + .getNodeService(NodeType.SESSION); + Map> zones = sessionNodeService + .fetchLoadbalanceMetrics(); + Map> zonesMaxConnectionsMap = new HashMap<>(); + for (String zoneName : zones.keySet()) { + Map nodes = zones.get(zoneName); + + Map maxConnectionsMap = zoneMaxConnections(nodes, maxDisconnect); + zonesMaxConnectionsMap.put(zoneName, maxConnectionsMap); + + sourceConnections.put(zoneName, nodes.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getConnectionCount()))); + } + sessionNodeService.configureLoadbalance(zonesMaxConnectionsMap); + result.put("source", sourceConnections); + result.put("target", zonesMaxConnectionsMap); + return result; + } + /** * Setter method for property taskListenerManager. * - * @param taskListenerManager value to be assigned to property taskListenerManager + * @param taskListenerManager value to be assigned to property taskListenerManager */ public void setTaskListenerManager(TaskListenerManager taskListenerManager) { this.taskListenerManager = taskListenerManager; @@ -397,7 +446,7 @@ public void setTaskListenerManager(TaskListenerManager taskListenerManager) { /** * Setter method for property dataStoreService. * - * @param dataStoreService value to be assigned to property dataStoreService + * @param dataStoreService value to be assigned to property dataStoreService */ public void setDataStoreService(StoreService dataStoreService) { this.dataStoreService = dataStoreService; @@ -406,7 +455,7 @@ public void setDataStoreService(StoreService dataStoreService) { /** * Setter method for property nodeConfig. * - * @param nodeConfig value to be assigned to property nodeConfig + * @param nodeConfig value to be assigned to property nodeConfig */ public void setNodeConfig(NodeConfig nodeConfig) { this.nodeConfig = nodeConfig; @@ -415,7 +464,7 @@ public void setNodeConfig(NodeConfig nodeConfig) { /** * Setter method for property sessionRepositoryService. * - * @param sessionRepositoryService value to be assigned to property sessionRepositoryService + * @param sessionRepositoryService value to be assigned to property sessionRepositoryService */ public void setSessionRepositoryService(RepositoryService> sessionRepositoryService) { this.sessionRepositoryService = sessionRepositoryService; @@ -424,7 +473,7 @@ public void setSessionRepositoryService(RepositoryServicesessionVersionRepositoryService. * - * @param sessionVersionRepositoryService value to be assigned to property sessionVersionRepositoryService + * @param sessionVersionRepositoryService value to be assigned to property sessionVersionRepositoryService */ public void setSessionVersionRepositoryService(VersionRepositoryService sessionVersionRepositoryService) { this.sessionVersionRepositoryService = sessionVersionRepositoryService; @@ -442,7 +491,7 @@ public NodeConfirmStatusService getSessionConfirmStatusService() { /** * Setter method for property sessionConfirmStatusService. * - * @param sessionConfirmStatusService value to be assigned to property sessionConfirmStatusService + * @param sessionConfirmStatusService value to be assigned to property sessionConfirmStatusService */ public void setSessionConfirmStatusService(NodeConfirmStatusService sessionConfirmStatusService) { this.sessionConfirmStatusService = sessionConfirmStatusService; diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/StoreService.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/StoreService.java index 05d38de1b..420e29dc5 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/StoreService.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/store/StoreService.java @@ -16,15 +16,15 @@ */ package com.alipay.sofa.registry.server.meta.store; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataCenterNodes; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import java.util.Collection; -import java.util.List; -import java.util.Map; - /** * interface for node store service * @author shangyu.wh @@ -72,7 +72,7 @@ public interface StoreService { * @param node * @return */ - void reNew(T node, int duration); + void renew(T node, int duration); /** * get expired node list diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/task/PersistenceDataChangeNotifyTask.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/task/PersistenceDataChangeNotifyTask.java index 63d986fcc..854c081a5 100644 --- a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/task/PersistenceDataChangeNotifyTask.java +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/task/PersistenceDataChangeNotifyTask.java @@ -16,10 +16,13 @@ */ package com.alipay.sofa.registry.server.meta.task; +import java.util.Set; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; import com.alipay.sofa.registry.server.meta.bootstrap.MetaServerConfig; import com.alipay.sofa.registry.server.meta.bootstrap.ServiceFactory; +import com.alipay.sofa.registry.server.meta.node.DataNodeService; import com.alipay.sofa.registry.server.meta.node.SessionNodeService; import com.alipay.sofa.registry.task.listener.TaskEvent; @@ -32,6 +35,8 @@ public class PersistenceDataChangeNotifyTask extends AbstractMetaServerTask { private final SessionNodeService sessionNodeService; + private final DataNodeService dataNodeService; + final private MetaServerConfig metaServerConfig; private NotifyProvideDataChange notifyProvideDataChange; @@ -40,11 +45,19 @@ public PersistenceDataChangeNotifyTask(MetaServerConfig metaServerConfig) { this.metaServerConfig = metaServerConfig; this.sessionNodeService = (SessionNodeService) ServiceFactory .getNodeService(NodeType.SESSION); + + this.dataNodeService = (DataNodeService) ServiceFactory.getNodeService(NodeType.DATA); } @Override public void execute() { - sessionNodeService.notifyProvideDataChange(notifyProvideDataChange); + Set nodeTypes = notifyProvideDataChange.getNodeTypes(); + if (nodeTypes.contains(NodeType.DATA)) { + dataNodeService.notifyProvideDataChange(notifyProvideDataChange); + } + if (nodeTypes.contains(NodeType.SESSION)) { + sessionNodeService.notifyProvideDataChange(notifyProvideDataChange); + } } @Override diff --git a/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/timertask/LogMetricsTask.java b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/timertask/LogMetricsTask.java new file mode 100644 index 000000000..8422cd9c0 --- /dev/null +++ b/server/server/meta/src/main/java/com/alipay/sofa/registry/server/meta/timertask/LogMetricsTask.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.timertask; + +import com.alipay.remoting.ProtocolCode; +import com.alipay.remoting.ProtocolManager; +import com.alipay.remoting.rpc.protocol.RpcProtocol; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.metrics.TaskMetrics; +import org.springframework.scheduling.annotation.Scheduled; + +import java.util.concurrent.ThreadPoolExecutor; + +public class LogMetricsTask { + private static final Logger EXE_LOGGER = LoggerFactory.getLogger("META-PROFILE-DIGEST", + "[ExecutorMetrics]"); + private final TaskMetrics taskMetrics = TaskMetrics.getInstance(); + + public LogMetricsTask() { + ThreadPoolExecutor boltDefaultExecutor = (ThreadPoolExecutor) ProtocolManager + .getProtocol(ProtocolCode.fromBytes(RpcProtocol.PROTOCOL_CODE)).getCommandHandler() + .getDefaultExecutor(); + taskMetrics.registerThreadExecutor("Meta-BoltDefaultExecutor", boltDefaultExecutor); + + } + + @Scheduled(initialDelayString = "${meta.server.metricsExecutor.fixedDelay}", fixedDelayString = "${meta.server.metricsExecutor.fixedDelay}") + public void printExecutorMetrics() { + EXE_LOGGER.info(TaskMetrics.getInstance().metricsString()); + } +} diff --git a/server/server/meta/src/main/resources/application.properties b/server/server/meta/src/main/resources/application.properties index 2449af5b3..ff31d74cf 100644 --- a/server/server/meta/src/main/resources/application.properties +++ b/server/server/meta/src/main/resources/application.properties @@ -1,6 +1,6 @@ spring.main.banner-mode=LOG -#nodes.metaNode=: -#nodes.localDataCenter= +nodes.metaNode=DefaultDataCenter:localhost +nodes.localDataCenter=DefaultDataCenter #meta.server.logging.level=INFO #meta.server.logging.home=/home/admin/logs/registry/meta @@ -10,3 +10,4 @@ meta.server.metaServerPort=9612 meta.server.raftServerPort=9614 meta.server.httpServerPort=9615 meta.server.raftGroup=MetaServerRaftGroup +meta.server.metricsExecutor.fixedDelay=30000 diff --git a/server/server/meta/src/main/resources/logback-spring.xml b/server/server/meta/src/main/resources/logback-spring.xml index 9251cda98..21cbf6b59 100644 --- a/server/server/meta/src/main/resources/logback-spring.xml +++ b/server/server/meta/src/main/resources/logback-spring.xml @@ -30,6 +30,25 @@ ${LOG_ENCODE} + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${META_LOG_HOME}/profile-digest.log + + ${META_LOG_HOME}/profile-digest.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + true @@ -173,6 +192,26 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${META_LOG_HOME}/registry-metrics.log + + ${META_LOG_HOME}/registry-metrics.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -213,6 +252,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/server/meta/src/main/resources/security/blacklist.txt b/server/server/meta/src/main/resources/security/blacklist.txt new file mode 100644 index 000000000..e3413f8d4 --- /dev/null +++ b/server/server/meta/src/main/resources/security/blacklist.txt @@ -0,0 +1,78 @@ +org.codehaus.groovy.runtime.MethodClosure +clojure.core$constantly +clojure.main$eval_opt +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactory +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactoryImpl +com.alibaba.citrus.springext.util.SpringExtUtil.AbstractProxy +com.alipay.custrelation.service.model.redress.Pair +com.caucho.hessian.test.TestCons +com.mchange.v2.c3p0.JndiRefForwardingDataSource +com.mchange.v2.c3p0.WrapperConnectionPoolDataSource +com.rometools.rome.feed.impl.EqualsBean +com.rometools.rome.feed.impl.ToStringBean +com.sun.jndi.rmi.registry.BindingEnumeration +com.sun.jndi.toolkit.dir.LazySearchEnumerationImpl +com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl +com.sun.rowset.JdbcRowSetImpl +com.sun.xml.internal.bind.v2.runtime.unmarshaller.Base64Data +java.rmi.server.UnicastRemoteObject +java.security.SignedObject +java.util.ServiceLoader$LazyIterator +javax.imageio.ImageIO$ContainsFilter +javax.imageio.spi.ServiceRegistry +javax.management.BadAttributeValueExpException +javax.naming.InitialContext +javax.naming.spi.ObjectFactory +javax.script.ScriptEngineManager +javax.sound.sampled.AudioFormat$Encoding +org.apache.carbondata.core.scan.expression.ExpressionResult +org.apache.commons.dbcp +org.apache.commons.dbcp2 +org.apache.commons.beanutils +org.apache.ibatis.executor.loader.AbstractSerialStateHolder +org.apache.ibatis.executor.loader.CglibSerialStateHolder +org.apache.ibatis.executor.loader.JavassistSerialStateHolder +org.apache.ibatis.executor.loader.cglib.CglibProxyFactory +org.apache.ibatis.executor.loader.javassist.JavassistSerialStateHolder +org.apache.tomcat.dbcp.dbcp.datasources.SharedPoolDataSource +org.apache.wicket.util.upload.DiskFileItem +org.apache.xalan.xsltc.trax.TemplatesImpl +org.apache.xbean.naming.context.ContextUtil$ReadOnlyBinding +org.apache.xpath.XPathContext +org.eclipse.jetty.util.log.LoggerLog +org.geotools.filter.ConstantExpression +org.springframework.aop.aspectj.autoproxy.AspectJAwareAdvisorAutoProxyCreator$PartiallyComparableAdvisorHolder +org.springframework.aop.support.DefaultBeanFactoryPointcutAdvisor +org.springframework.beans.factory.BeanFactory +org.springframework.beans.factory.config.PropertyPathFactoryBean +org.springframework.beans.factory.support.DefaultListableBeanFactory +org.springframework.jndi.support.SimpleJndiBeanFactory +org.springframework.orm.jpa.AbstractEntityManagerFactoryBean +org.springframework.transaction.jta.JtaTransactionManager +org.yaml.snakeyaml.tokens.DirectiveToken +sun.rmi.server.UnicastRef +javax.management.ImmutableDescriptor +org.springframework.jndi.JndiObjectTargetSource +ch.qos.logback.core.db.JNDIConnectionSource +java.beans.Expression +javassist.bytecode +javassist.tools.web.Viewer +javassist.util.proxy.SerializedProxy +org.apache.ibatis.javassist.bytecode +org.apache.ibatis.javassist.tools.web.Viewer +org.apache.ibatis.javassist.util.proxy.SerializedProxy +org.springframework.beans.factory.config.MethodInvokingFactoryBean +com.alibaba.druid +com.sun.corba +com.sun.org.apache.bcel +com.sun.org.apache.xml +com.sun.org.apache.xpath +org.apache.zookeeper.Shell +org.apache.tomcat.dbcp.dbcp.BasicDataSource +net.bytebuddy.dynamic.loading.ByteArrayClassLoader +org.jboss.resteasy.plugins +org.springframework.beans.BeanWrapperImpl$BeanPropertyHandler +org.apache.velocity.runtime.resource.Resource +org.springframework.expression.spel.ast.Indexer$PropertyIndexingValueRef +org.mortbay.log.Slf4jLog +org.springframework.expression.spel.ast.MethodReference$MethodValueRef \ No newline at end of file diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/DataNodeChangePushTaskListenerMock.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/DataNodeChangePushTaskListenerMock.java index 162df86cd..609415090 100644 --- a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/DataNodeChangePushTaskListenerMock.java +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/DataNodeChangePushTaskListenerMock.java @@ -16,6 +16,11 @@ */ package com.alipay.sofa.registry.server.meta.test.confirm; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.server.meta.store.DataStoreService; @@ -24,11 +29,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * * @author shangyu.wh @@ -47,8 +47,8 @@ public DataNodeChangePushTaskListenerMock(DataStoreService dataStoreService, } @Override - public boolean support(TaskEvent event) { - return TaskType.DATA_NODE_CHANGE_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.DATA_NODE_CHANGE_PUSH_TASK; } @Override diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/ReceiveStatusConfirmNotifyTaskMock.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/ReceiveStatusConfirmNotifyTaskMock.java index bbdf33c62..5d2dcc302 100644 --- a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/ReceiveStatusConfirmNotifyTaskMock.java +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/ReceiveStatusConfirmNotifyTaskMock.java @@ -34,8 +34,8 @@ public class ReceiveStatusConfirmNotifyTaskMock implements TaskListener { ReceiveStatusConfirmNotifyTaskMock.class, "[Task]"); @Override - public boolean support(TaskEvent event) { - return TaskType.RECEIVE_STATUS_CONFIRM_NOTIFY_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.RECEIVE_STATUS_CONFIRM_NOTIFY_TASK; } @Override diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/SessionNodeChangePushTaskMock.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/SessionNodeChangePushTaskMock.java index 8861c623a..35f5e4840 100644 --- a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/SessionNodeChangePushTaskMock.java +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/confirm/SessionNodeChangePushTaskMock.java @@ -16,6 +16,11 @@ */ package com.alipay.sofa.registry.server.meta.test.confirm; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import com.alipay.sofa.registry.common.model.metaserver.SessionNode; import com.alipay.sofa.registry.server.meta.store.SessionStoreService; import com.alipay.sofa.registry.server.meta.task.Constant; @@ -23,11 +28,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - /** * * @author shangyu.wh @@ -46,8 +46,8 @@ public SessionNodeChangePushTaskMock(SessionStoreService sessionStoreService, } @Override - public boolean support(TaskEvent event) { - return TaskType.SESSION_NODE_CHANGE_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.SESSION_NODE_CHANGE_PUSH_TASK; } @Override diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/DataRepositoryServiceTest.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/DataRepositoryServiceTest.java new file mode 100644 index 000000000..d8a2fd1ed --- /dev/null +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/DataRepositoryServiceTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.test.service; + +import com.alipay.sofa.registry.common.model.Node.NodeStatus; +import com.alipay.sofa.registry.common.model.metaserver.DataNode; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; +import com.alipay.sofa.registry.server.meta.repository.NodeRepository; +import com.alipay.sofa.registry.server.meta.repository.service.DataRepositoryService; +import com.alipay.sofa.registry.server.meta.store.RenewDecorate; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * @author shangyu.wh + * @version 1.0: MetaRepositoryServiceTest.java, v 0.1 2019-08-01 12:18 shangyu.wh Exp $ + */ +public class DataRepositoryServiceTest { + + @Test + public void testDataRepositoryServicePut() { + Map registry = new ConcurrentHashMap<>(); + DataRepositoryService metaRepositoryService = new DataRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new DataNode(new URL(ip, 0), + dataCenter)); + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getIp(), ip); + } + + @Test + public void testDataRepositoryServiceRemove() { + Map registry = new ConcurrentHashMap<>(); + DataRepositoryService metaRepositoryService = new DataRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new DataNode(new URL(ip, 0), + dataCenter)); + + metaRepositoryService.put(ip, metaNode); + + metaRepositoryService.remove(ip); + + Assert.assertEquals(metaRepositoryService.getAllData().size(), 0); + } + + @Test + public void testMetaRepositoryServiceReplace() { + Map registry = new ConcurrentHashMap<>(); + DataRepositoryService metaRepositoryService = new DataRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new DataNode(new URL(ip, 0), + dataCenter)); + + DataNode metaNodeFix = new DataNode(new URL(ip, 0), dataCenter); + metaNodeFix.setNodeStatus(NodeStatus.WORKING); + + RenewDecorate metaNode2 = new RenewDecorate(metaNodeFix); + + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.INIT); + + metaRepositoryService.replace(ip, metaNode2); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.WORKING); + Assert.assertEquals(metaRepositoryService.getAllData().size(), 1); + } + + @Test + public void testMetaRepositoryServiceReplaceAll() { + Map registry = new ConcurrentHashMap<>(); + DataRepositoryService metaRepositoryService = new DataRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new DataNode(new URL(ip, 0), + dataCenter)); + + DataNode metaNodeFix = new DataNode(new URL(ip, 0), dataCenter); + metaNodeFix.setNodeStatus(NodeStatus.WORKING); + + RenewDecorate metaNode2 = new RenewDecorate(metaNodeFix); + + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.INIT); + + Map> map = new HashMap<>(); + map.put(ip, metaNode2); + + metaRepositoryService.replaceAll(dataCenter, map, 1234l); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.WORKING); + Assert.assertEquals(metaRepositoryService.getAllDataMap().get(dataCenter).size(), 1); + } +} \ No newline at end of file diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/MetaRepositoryServiceTest.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/MetaRepositoryServiceTest.java new file mode 100644 index 000000000..479012abd --- /dev/null +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/MetaRepositoryServiceTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.test.service; + +import com.alipay.sofa.registry.common.model.Node.NodeStatus; +import com.alipay.sofa.registry.common.model.metaserver.MetaNode; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; +import com.alipay.sofa.registry.server.meta.repository.NodeRepository; +import com.alipay.sofa.registry.server.meta.repository.service.MetaRepositoryService; +import com.alipay.sofa.registry.server.meta.store.RenewDecorate; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * @author shangyu.wh + * @version 1.0: MetaRepositoryServiceTest.java, v 0.1 2019-08-01 12:18 shangyu.wh Exp $ + */ +public class MetaRepositoryServiceTest { + + @Test + public void testMetaRepositoryServicePut() { + Map registry = new ConcurrentHashMap<>(); + MetaRepositoryService metaRepositoryService = new MetaRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new MetaNode(new URL(ip, 0), + dataCenter)); + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getIp(), ip); + } + + @Test + public void testMetaRepositoryServiceRemove() { + Map registry = new ConcurrentHashMap<>(); + MetaRepositoryService metaRepositoryService = new MetaRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new MetaNode(new URL(ip, 0), + dataCenter)); + + metaRepositoryService.put(ip, metaNode); + + metaRepositoryService.remove(ip); + + Assert.assertEquals(metaRepositoryService.getAllData().size(), 0); + } + + @Test + public void testMetaRepositoryServiceReplace() { + Map registry = new ConcurrentHashMap<>(); + MetaRepositoryService metaRepositoryService = new MetaRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new MetaNode(new URL(ip, 0), + dataCenter)); + + MetaNode metaNodeFix = new MetaNode(new URL(ip, 0), dataCenter); + metaNodeFix.setNodeStatus(NodeStatus.WORKING); + + RenewDecorate metaNode2 = new RenewDecorate(metaNodeFix); + + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.INIT); + + metaRepositoryService.replace(ip, metaNode2); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.WORKING); + Assert.assertEquals(metaRepositoryService.getAllData().size(), 1); + } + + @Test + public void testMetaRepositoryServiceReplaceAll() { + Map registry = new ConcurrentHashMap<>(); + MetaRepositoryService metaRepositoryService = new MetaRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + metaRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate metaNode = new RenewDecorate(new MetaNode(new URL(ip, 0), + dataCenter)); + + MetaNode metaNodeFix = new MetaNode(new URL(ip, 0), dataCenter); + metaNodeFix.setNodeStatus(NodeStatus.WORKING); + + RenewDecorate metaNode2 = new RenewDecorate(metaNodeFix); + + metaRepositoryService.put(ip, metaNode); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.INIT); + + Map> map = new HashMap<>(); + map.put(ip, metaNode2); + + metaRepositoryService.replaceAll(dataCenter, map, 1234l); + + Assert.assertEquals(metaRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.WORKING); + Assert.assertEquals(metaRepositoryService.getAllDataMap().get(dataCenter).size(), 1); + } +} \ No newline at end of file diff --git a/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/SessionRepositoryServiceTest.java b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/SessionRepositoryServiceTest.java new file mode 100644 index 000000000..da0997188 --- /dev/null +++ b/server/server/meta/src/test/java/com/alipay/sofa/registry/server/meta/test/service/SessionRepositoryServiceTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.meta.test.service; + +import com.alipay.sofa.registry.common.model.Node.NodeStatus; +import com.alipay.sofa.registry.common.model.metaserver.SessionNode; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.server.meta.bootstrap.NodeConfig; +import com.alipay.sofa.registry.server.meta.repository.service.SessionRepositoryService; +import com.alipay.sofa.registry.server.meta.store.RenewDecorate; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * @author shangyu.wh + * @version 1.0: SessionRepositoryServiceTest.java, v 0.1 2019-08-01 13:04 shangyu.wh Exp $ + */ +public class SessionRepositoryServiceTest { + + @Test + public void testSessionRepositoryServicePut() { + ConcurrentHashMap> registry = new ConcurrentHashMap<>(); + SessionRepositoryService sessionRepositoryService = new SessionRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + sessionRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate sessionNode = new RenewDecorate(new SessionNode(new URL(ip, 0), + dataCenter)); + sessionRepositoryService.put(ip, sessionNode); + + Assert.assertEquals(sessionRepositoryService.get(ip).getRenewal().getIp(), ip); + } + + @Test + public void testSessionRepositoryServiceRemove() { + ConcurrentHashMap> registry = new ConcurrentHashMap<>(); + SessionRepositoryService sessionRepositoryService = new SessionRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + sessionRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate sessionNode = new RenewDecorate(new SessionNode(new URL(ip, 0), + dataCenter)); + sessionRepositoryService.put(ip, sessionNode); + + sessionRepositoryService.remove(ip); + + Assert.assertEquals(sessionRepositoryService.getAllData().size(), 0); + } + + @Test + public void testSessionRepositoryServiceReplace() { + ConcurrentHashMap> registry = new ConcurrentHashMap<>(); + SessionRepositoryService sessionRepositoryService = new SessionRepositoryService(registry); + + String ip = "192.1.1.1"; + String dataCenter = "zue"; + + sessionRepositoryService.setNodeConfig(new NodeConfig() { + @Override + public Map> getMetaNode() { + return null; + } + + @Override + public Map> getMetaNodeIP() { + return null; + } + + @Override + public String getLocalDataCenter() { + return dataCenter; + } + + @Override + public String getMetaDataCenter(String metaIpAddress) { + return dataCenter; + } + + @Override + public Set getDataCenterMetaServers(String dataCenter) { + return null; + } + }); + + RenewDecorate sessionNod = new RenewDecorate(new SessionNode(new URL(ip, 0), + dataCenter)); + + SessionNode metaNodeFix = new SessionNode(new URL(ip, 0), dataCenter); + metaNodeFix.setNodeStatus(NodeStatus.WORKING); + + RenewDecorate metaNode2 = new RenewDecorate(metaNodeFix); + + sessionRepositoryService.put(ip, sessionNod); + + Assert.assertEquals(sessionRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.INIT); + + sessionRepositoryService.replace(ip, metaNode2); + + Assert.assertEquals(sessionRepositoryService.get(ip).getRenewal().getNodeStatus(), + NodeStatus.WORKING); + Assert.assertEquals(sessionRepositoryService.getAllData().size(), 1); + Assert.assertEquals(sessionRepositoryService.getAllDataMap().get(dataCenter).size(), 1); + } + +} \ No newline at end of file diff --git a/server/server/meta/src/test/resources/application.properties b/server/server/meta/src/test/resources/application.properties index 386132e80..76744cd40 100644 --- a/server/server/meta/src/test/resources/application.properties +++ b/server/server/meta/src/test/resources/application.properties @@ -6,4 +6,5 @@ meta.server.metaServerPort=9612 meta.server.raftServerPort=9614 meta.server.httpServerPort=9615 meta.server.raftGroup=MetaServerRaftGroup -meta.server.decisionMode=RUNTIME \ No newline at end of file +meta.server.decisionMode=RUNTIME +meta.server.metricsExecutor.fixedDelay=30000 \ No newline at end of file diff --git a/server/server/pom.xml b/server/server/pom.xml index 30a7de600..7fe829ef7 100644 --- a/server/server/pom.xml +++ b/server/server/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/server/session/pom.xml b/server/server/session/pom.xml index 25d435d51..2a2f43725 100644 --- a/server/server/session/pom.xml +++ b/server/server/session/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 @@ -17,6 +17,10 @@ + + org.slf4j + jul-to-slf4j + org.springframework.boot spring-boot-starter @@ -74,6 +78,11 @@ commons-lang commons-lang + + junit + junit + test + diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptor.java new file mode 100644 index 000000000..3f844b234 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptor.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.acceptor; + +/** + * + * @author kezhu.wukz + * @author shangyu.wh + * @version 1.0: WriteDataAcceptor.java, v 0.1 2019-06-11 15:08 shangyu.wh Exp $ + */ +public interface WriteDataAcceptor { + + /** + * accept all write data request + * @param request + */ + void accept(WriteDataRequest request); + + /** + * remove processor of connectId + * @param connectId + */ + void remove(String connectId); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptorImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptorImpl.java new file mode 100644 index 000000000..7dd839c01 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataAcceptorImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.acceptor; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.renew.RenewService; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; + +/** + * + * @author kezhu.wukz + * @author shangyu.wh + * @version 1.0: WriteDataAcceptor.java, v 0.1 2019-06-06 12:45 shangyu.wh Exp $ + */ +public class WriteDataAcceptorImpl implements WriteDataAcceptor { + + @Autowired + private TaskListenerManager taskListenerManager; + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + private RenewService renewService; + + /** + * acceptor for all write data request + * key:connectId + * value:writeRequest processor + * + */ + private Map writeDataProcessors = new ConcurrentHashMap(); + + public void accept(WriteDataRequest request) { + String connectId = request.getConnectId(); + WriteDataProcessor writeDataProcessor = writeDataProcessors.computeIfAbsent(connectId, + key -> new WriteDataProcessor(connectId, taskListenerManager, sessionServerConfig, renewService)); + + writeDataProcessor.process(request); + } + + public void remove(String connectId) { + writeDataProcessors.remove(connectId); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataProcessor.java new file mode 100644 index 000000000..3e9e497b5 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataProcessor.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.acceptor; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.acceptor.WriteDataRequest.WriteDataRequestType; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.renew.RenewService; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import com.google.common.collect.Lists; + +/** + * + * @author kezhu.wukz + * @author shangyu.wh + * @version 1.0: WriteDataProcessor.java, v 0.1 2019-06-06 12:50 shangyu.wh Exp $ + */ +public class WriteDataProcessor { + + private static final Logger LOGGER = LoggerFactory + .getLogger(WriteDataProcessor.class); + + private static final Logger RENEW_LOGGER = LoggerFactory + .getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[WriteDataProcessor]"); + + private static final Logger taskLogger = LoggerFactory + .getLogger( + WriteDataProcessor.class, + "[Task]"); + + private final TaskListenerManager taskListenerManager; + + private final SessionServerConfig sessionServerConfig; + + private final RenewService renewService; + + private final String connectId; + + private Map lastUpdateTimestampMap = new ConcurrentHashMap<>(); + + private AtomicBoolean writeDataLock = new AtomicBoolean( + false); + + private ConcurrentLinkedQueue acceptorQueue = new ConcurrentLinkedQueue(); + + private AtomicInteger acceptorQueueSize = new AtomicInteger(0); + + public WriteDataProcessor(String connectId, TaskListenerManager taskListenerManager, + SessionServerConfig sessionServerConfig, RenewService renewService) { + this.connectId = connectId; + this.taskListenerManager = taskListenerManager; + this.sessionServerConfig = sessionServerConfig; + this.renewService = renewService; + } + + private boolean halt() { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("halt: connectId={}", connectId); + } + return writeDataLock.compareAndSet(false, true); + } + + public void resume() { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("resume: connectId={}", connectId); + } + flushQueue(); + writeDataLock.compareAndSet(true, false); + flushQueue(); + } + + public void process(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("process: connectId={}, requestType={}, requestBody={}", connectId, + request.getRequestType(), request.getRequestBody()); + } + + // record the last update time by pub/unpub + if (isWriteRequest(request)) { + refreshUpdateTime(request.getDataServerIP()); + } + + if (request.getRequestType() == WriteDataRequestType.DATUM_SNAPSHOT) { + // snapshot has high priority, so handle directly + doHandle(request); + } else { + // If locked, insert the queue; + // otherwise, try emptying the queue (to avoid residue) before processing the request. + if (writeDataLock.get()) { + addQueue(request); + } else { + flushQueue(); + doHandle(request); + } + } + + } + + private void addQueue(WriteDataRequest request) { + if (acceptorQueueSize.incrementAndGet() <= sessionServerConfig + .getWriteDataAcceptorQueueSize()) { + acceptorQueue.add(request); + } else { + RENEW_LOGGER + .error( + "acceptorQueueSize({}) reached the limit : connectId={}, requestType={}, requestBody={}", + acceptorQueue.size(), connectId, request.getRequestType(), + request.getRequestBody()); + } + } + + /** + * + * @param request + * @return + */ + private boolean isWriteRequest(WriteDataRequest request) { + return request.getRequestType() == WriteDataRequestType.PUBLISHER + || request.getRequestType() == WriteDataRequestType.UN_PUBLISHER; + } + + /** + * Ensure that the queue data is sent out + */ + private void flushQueue() { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("flushQueue: connectId={}", connectId); + } + + while (!acceptorQueue.isEmpty()) { + WriteDataRequest writeDataRequest = acceptorQueue.poll(); + if (writeDataRequest == null) { + break; + } + acceptorQueueSize.decrementAndGet(); + doHandle(writeDataRequest); + } + } + + private void doHandle(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doHandle: connectId={}, requestType={}, requestBody={}", connectId, + request.getRequestType(), request.getRequestBody()); + } + + switch (request.getRequestType()) { + case PUBLISHER: { + doPublishAsync(request); + } + break; + case UN_PUBLISHER: { + doUnPublishAsync(request); + } + break; + case CLIENT_OFF: { + doClientOffAsync(request); + } + break; + case RENEW_DATUM: { + if (renewAndSnapshotInSilence(request.getDataServerIP())) { + return; + } + doRenewAsync(request); + } + break; + case DATUM_SNAPSHOT: { + if (renewAndSnapshotInSilenceAndRefreshUpdateTime(request.getDataServerIP())) { + return; + } + halt(); + try { + doSnapshotAsync(request); + } finally { + resume(); + } + } + break; + default: + LOGGER.warn("Unknown request type, requestType={}, requestBody={}", connectId, + request.getRequestType(), request.getRequestBody()); + + } + } + + private void doRenewAsync(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doRenewAsync: connectId={}, requestType={}, requestBody={}", + connectId, request.getRequestType(), request.getRequestBody()); + } + + sendEvent(request.getRequestBody(), TaskType.RENEW_DATUM_TASK); + } + + private void doClientOffAsync(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doClientOffAsync: connectId={}, requestType={}, requestBody={}", + connectId, request.getRequestType(), request.getRequestBody()); + } + + String connectId = request.getConnectId(); + sendEvent(Lists.newArrayList(connectId), TaskType.CANCEL_DATA_TASK); + } + + private void doUnPublishAsync(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doUnPublishAsync: connectId={}, requestType={}, requestBody={}", + connectId, request.getRequestType(), request.getRequestBody()); + } + + sendEvent(request.getRequestBody(), TaskType.UN_PUBLISH_DATA_TASK); + } + + private void doPublishAsync(WriteDataRequest request) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("doPublishAsync: connectId={}, requestType={}, requestBody={}", + connectId, request.getRequestType(), request.getRequestBody()); + } + + sendEvent(request.getRequestBody(), TaskType.PUBLISH_DATA_TASK); + } + + private void sendEvent(Object eventObj, TaskType taskType) { + TaskEvent taskEvent = new TaskEvent(eventObj, taskType); + + //print but ignore if from renew module, cause renew request is too much + if (taskType != TaskType.RENEW_DATUM_TASK) { + taskLogger.info("send " + taskType + " taskEvent:{}", taskEvent); + } + + taskListenerManager.sendTaskEvent(taskEvent); + } + + private void doSnapshotAsync(WriteDataRequest request) { + RENEW_LOGGER.info( + "doSnapshotAsync: connectId={}, dataServerIP={}, requestType={}, requestBody={}", + connectId, request.getDataServerIP(), request.getRequestType(), + request.getRequestBody()); + + String connectId = (String) request.getRequestBody(); + DatumSnapshotRequest datumSnapshotRequest = renewService.getDatumSnapshotRequest(connectId, + request.getDataServerIP()); + if (datumSnapshotRequest != null) { + TaskEvent taskEvent = new TaskEvent(datumSnapshotRequest, TaskType.DATUM_SNAPSHOT_TASK); + taskListenerManager.sendTaskEvent(taskEvent); + } else { + RENEW_LOGGER + .info( + "datumSnapshotRequest is null when doSnapshotAsync: connectId={}, dataServerIP={}, requestType={}", + connectId, request.getDataServerIP(), request.getRequestType()); + } + + } + + /** + * In silence, do not renew and snapshot + */ + private boolean renewAndSnapshotInSilence(String dataServerIP) { + boolean renewAndSnapshotInSilence = System.currentTimeMillis() + - getLastUpdateTime(dataServerIP).get() < this.sessionServerConfig + .getRenewAndSnapshotSilentPeriodSec() * 1000L; + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug( + "renewAndSnapshotInSilence: connectId={}, renewAndSnapshotInSilence={}", connectId, + renewAndSnapshotInSilence); + } + return renewAndSnapshotInSilence; + } + + /** + * In silence, do not renew and snapshot + */ + private boolean renewAndSnapshotInSilenceAndRefreshUpdateTime(String dataServerIP) { + boolean renewAndSnapshotInSilence = System.currentTimeMillis() + - refreshUpdateTime(dataServerIP) < this.sessionServerConfig + .getRenewAndSnapshotSilentPeriodSec() * 1000L; + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER + .debug( + "renewAndSnapshotInSilenceAndRefreshUpdateTime: connectId={}, renewAndSnapshotInSilence={}", + connectId, renewAndSnapshotInSilence); + } + return renewAndSnapshotInSilence; + } + + private long refreshUpdateTime(String dataServerIP) { + AtomicLong lastUpdateTimestamp = getLastUpdateTime(dataServerIP); + return lastUpdateTimestamp.getAndSet(System.currentTimeMillis()); + } + + private AtomicLong getLastUpdateTime(String dataServerIP) { + AtomicLong lastUpdateTimestamp = lastUpdateTimestampMap.get(dataServerIP); + if (lastUpdateTimestamp == null) { + lastUpdateTimestamp = new AtomicLong(0); + AtomicLong _lastUpdateTimestamp = lastUpdateTimestampMap.putIfAbsent(dataServerIP, + lastUpdateTimestamp); + if (_lastUpdateTimestamp != null) { + lastUpdateTimestamp = _lastUpdateTimestamp; + } + } + return lastUpdateTimestamp; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataRequest.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataRequest.java new file mode 100644 index 000000000..0e71f8ab1 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/acceptor/WriteDataRequest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.acceptor; + +/** + * + * @author kezhu.wukz + * @author shangyu.wh + * @version 1.0: WriteDataRequest.java, v 0.1 2019-06-06 18:42 shangyu.wh Exp $ + */ +public interface WriteDataRequest { + + /** + * The enum for request type + */ + enum WriteDataRequestType { + PUBLISHER, UN_PUBLISHER, CLIENT_OFF, RENEW_DATUM, DATUM_SNAPSHOT + } + + /** + * Gets request body. + * + * @return the request body + */ + T getRequestBody(); + + WriteDataRequestType getRequestType(); + + String getConnectId(); + + String getDataServerIP(); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/CommonConfig.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/CommonConfig.java index 71e4f81a2..7ae7a8cec 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/CommonConfig.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/CommonConfig.java @@ -16,11 +16,13 @@ */ package com.alipay.sofa.registry.server.session.bootstrap; -import org.springframework.beans.factory.annotation.Value; - import java.util.Collection; import java.util.Map; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.springframework.beans.factory.annotation.Value; + /** * * @author shangyu.wh @@ -85,4 +87,9 @@ public String getLocalRegion() { return localRegion; } + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java index 87b01b6bb..0b2057bd9 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java @@ -16,7 +16,20 @@ */ package com.alipay.sofa.registry.server.session.bootstrap; -import com.alipay.sofa.registry.common.model.Node; +import java.lang.annotation.Annotation; +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Resource; +import javax.ws.rs.Path; +import javax.ws.rs.ext.Provider; + +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; + import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.metaserver.FetchProvideDataRequest; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; @@ -31,27 +44,15 @@ import com.alipay.sofa.registry.remoting.Server; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistManager; import com.alipay.sofa.registry.server.session.node.NodeManager; import com.alipay.sofa.registry.server.session.node.NodeManagerFactory; import com.alipay.sofa.registry.server.session.node.RaftClientManager; import com.alipay.sofa.registry.server.session.node.SessionProcessIdGenerator; -import com.alipay.sofa.registry.server.session.remoting.handler.AbstractClientHandler; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; import com.alipay.sofa.registry.server.session.remoting.handler.AbstractServerHandler; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.task.batcher.TaskDispatchers; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.util.CollectionUtils; - -import javax.annotation.Resource; -import javax.ws.rs.Path; -import javax.ws.rs.ext.Provider; -import java.lang.annotation.Annotation; -import java.util.Collection; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; /** * The type Session server bootstrap. @@ -78,18 +79,15 @@ public class SessionServerBootstrap { @Resource(name = "serverHandlers") private Collection serverHandlers; - @Resource(name = "dataClientHandlers") - private Collection dataClientHandlers; - - @Autowired - private NodeManager dataNodeManager; - @Autowired private NodeManager metaNodeManager; @Autowired protected NodeExchanger metaNodeExchanger; + @Autowired + private NodeExchanger dataNodeExchanger; + @Autowired private ResourceConfig jerseyResourceConfig; @@ -99,12 +97,16 @@ public class SessionServerBootstrap { @Autowired private RaftClientManager raftClientManager; + @Autowired + private BlacklistManager blacklistManager; + + @Autowired + private ProvideDataProcessor provideDataProcessorManager; + private Server server; private Server httpServer; - private Client dataClient; - private Client metaClient; private AtomicBoolean metaStart = new AtomicBoolean(false); @@ -120,8 +122,10 @@ public class SessionServerBootstrap { /** * Do initialized. */ - public void doInitialized() { + public void start() { try { + LOGGER.info("the configuration items are as follows: " + sessionServerConfig.toString()); + initEnvironment(); startRaftClient(); @@ -158,7 +162,6 @@ private void doStop() { executorManager.stopScheduler(); TaskDispatchers.stopDefaultSingleTaskDispatcher(); - closeClients(); stopHttpServer(); stopServer(); } catch (Throwable e) { @@ -208,28 +211,7 @@ private void openSessionServer() { private void connectDataServer() { try { if (dataStart.compareAndSet(false, true)) { - Collection dataNodes = dataNodeManager.getDataCenterNodes(); - if (CollectionUtils.isEmpty(dataNodes)) { - dataNodeManager.getAllDataCenterNodes(); - dataNodes = dataNodeManager.getDataCenterNodes(); - } - if (!CollectionUtils.isEmpty(dataNodes)) { - for (Node dataNode : dataNodes) { - if (dataNode.getNodeUrl() == null - || dataNode.getNodeUrl().getIpAddress() == null) { - LOGGER - .error("get data node address error!url{}", dataNode.getNodeUrl()); - continue; - } - dataClient = boltExchange.connect( - Exchange.DATA_SERVER_TYPE, - new URL(dataNode.getNodeUrl().getIpAddress(), sessionServerConfig - .getDataServerPort()), dataClientHandlers - .toArray(new ChannelHandler[dataClientHandlers.size()])); - } - LOGGER.info("Data server connected {} server! port:{}", dataNodes.size(), - sessionServerConfig.getDataServerPort()); - } + dataNodeExchanger.connectServer(); } } catch (Exception e) { dataStart.set(false); @@ -249,8 +231,6 @@ private void connectMetaServer() { if (metaStart.compareAndSet(false, true)) { metaClient = metaNodeExchanger.connectServer(); - int size = metaClient.getChannels().size(); - URL leaderUrl = new URL(raftClientManager.getLeader().getIp(), sessionServerConfig.getMetaServerPort()); @@ -260,7 +240,11 @@ private void connectMetaServer() { fetchStopPushSwitch(leaderUrl); - LOGGER.info("MetaServer connected {} server! Port:{}", size, + fetchEnableDataRenewSnapshot(leaderUrl); + + fetchBlackList(); + + LOGGER.info("MetaServer connected meta server! Port:{}", sessionServerConfig.getMetaServerPort()); } } catch (Exception e) { @@ -293,35 +277,36 @@ private void fetchStopPushSwitch(URL leaderUrl) { Object ret = sendMetaRequest(fetchProvideDataRequest, leaderUrl); if (ret instanceof ProvideData) { ProvideData provideData = (ProvideData) ret; - if (provideData.getProvideData() == null - || provideData.getProvideData().getObject() == null) { - LOGGER.info("Fetch session stop push switch no data existed,config not change!"); - return; - } - String data = (String) provideData.getProvideData().getObject(); - sessionServerConfig.setStopPushSwitch(Boolean.valueOf(data)); - if (data != null) { - if (!Boolean.valueOf(data)) { - //stop push init on,then begin fetch data schedule task - sessionServerConfig.setBeginDataFetchTask(true); - } - } - LOGGER.info("Fetch session stop push data switch {} success!", data); + provideDataProcessorManager.fetchDataProcess(provideData); } else { LOGGER.info("Fetch session stop push switch data null,config not change!"); } } + private void fetchEnableDataRenewSnapshot(URL leaderUrl) { + FetchProvideDataRequest fetchProvideDataRequest = new FetchProvideDataRequest( + ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT); + Object data = sendMetaRequest(fetchProvideDataRequest, leaderUrl); + if (data instanceof ProvideData) { + ProvideData provideData = (ProvideData) data; + provideDataProcessorManager.fetchDataProcess(provideData); + } + } + + private void fetchBlackList() { + blacklistManager.load(); + } + private Object sendMetaRequest(Object request, URL leaderUrl) { Object ret; try { - ret = metaClient.sendSync(metaClient.getChannel(leaderUrl), request, + ret = metaClient.sendSync(leaderUrl, request, sessionServerConfig.getMetaNodeExchangeTimeOut()); } catch (Exception e) { URL leaderUrlNew = new URL(raftClientManager.refreshLeader().getIp(), sessionServerConfig.getMetaServerPort()); LOGGER.warn("request send error!It will be retry once to new leader {}!", leaderUrlNew); - ret = metaClient.sendSync(metaClient.getChannel(leaderUrlNew), request, + ret = metaClient.sendSync(leaderUrlNew, request, sessionServerConfig.getMetaNodeExchangeTimeOut()); } return ret; @@ -372,12 +357,6 @@ private void stopServer() { } } - private void closeClients() { - if (dataClient != null && !dataClient.isClosed()) { - dataClient.close(); - } - } - private void stopHttpServer() { if (httpServer != null && httpServer.isOpen()) { httpServer.close(); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfig.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfig.java index 98668f574..371858760 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfig.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfig.java @@ -53,20 +53,44 @@ public interface SessionServerConfig { int getDataNodeExchangeTimeOut(); + int getDataNodeExchangeForFetchDatumTimeOut(); + int getMetaNodeExchangeTimeOut(); String getSessionServerRegion(); + String getClientCell(String clientCell); + String getSessionServerDataCenter(); int getReceivedDataMultiPushTaskRetryTimes(); int getCancelDataTaskRetryTimes(); - int getCancelDataTaskRetryFirstDelay(); + long getCancelDataTaskRetryFirstDelay(); long getCancelDataTaskRetryIncrementDelay(); + int getPublishDataTaskRetryTimes(); + + long getPublishDataTaskRetryFirstDelay(); + + long getPublishDataTaskRetryIncrementDelay(); + + int getUnPublishDataTaskRetryTimes(); + + long getUnPublishDataTaskRetryFirstDelay(); + + long getUnPublishDataTaskRetryIncrementDelay(); + + int getDatumSnapshotTaskRetryTimes(); + + long getDatumSnapshotTaskRetryFirstDelay(); + + long getDatumSnapshotTaskRetryIncrementDelay(); + + int getRenewDatumTaskRetryTimes(); + int getDataChangeFetchTaskRetryTimes(); int getSubscriberRegisterFetchRetryTimes(); @@ -85,6 +109,12 @@ public interface SessionServerConfig { int getSchedulerConnectDataExpBackOffBound(); + int getSchedulerCleanInvalidClientTimeOut(); + + int getSchedulerCleanInvalidClientFirstDelay(); + + int getSchedulerCleanInvalidClientBackOffBound(); + int getAccessDataExecutorMinPoolSize(); int getAccessDataExecutorMaxPoolSize(); @@ -109,11 +139,11 @@ public interface SessionServerConfig { long getPushTaskExecutorKeepAliveTime(); - int getDisconnectClientExecutorMinPoolSize(); + int getConnectClientExecutorMinPoolSize(); - int getDisconnectClientExecutorMaxPoolSize(); + int getConnectClientExecutorMaxPoolSize(); - int getDisconnectClientExecutorQueueSize(); + int getConnectClientExecutorQueueSize(); int getDataChangeFetchTaskMaxBufferSize(); @@ -123,10 +153,26 @@ public interface SessionServerConfig { int getUserDataPushRetryWheelTicksDuration(); + int getUserDataPushRetryExecutorQueueSize(); + + int getUserDataPushRetryExecutorThreadSize(); + int getPushDataTaskRetryFirstDelay(); long getPushDataTaskRetryIncrementDelay(); + int getRenewDatumWheelTicksSize(); + + int getRenewDatumWheelTicksDuration(); + + int getRenewDatumWheelTaskDelaySec(); + + int getRenewDatumWheelTaskRandomFirstDelaySec(); + + String getBlacklistPubDataIdRegex(); + + String getBlacklistSubDataIdRegex(); + int getNumberOfReplicas(); boolean isStopPushSwitch(); @@ -140,4 +186,50 @@ public interface SessionServerConfig { boolean isInvalidForeverZone(String zoneId); boolean isInvalidIgnored(String dataId); + + int getRenewAndSnapshotSilentPeriodSec(); + + int getWriteDataAcceptorQueueSize(); + + int getDataNodeRetryExecutorQueueSize(); + + int getDataNodeRetryExecutorThreadSize(); + + int getRenewDatumWheelThreadSize(); + + int getRenewDatumWheelQueueSize(); + + long getPushTaskConfirmWaitTimeout(); + + int getPushTaskConfirmCheckWheelTicksSize(); + + int getPushTaskConfirmCheckWheelTicksDuration(); + + int getPushTaskConfirmCheckExecutorQueueSize(); + + int getPushTaskConfirmCheckExecutorThreadSize(); + + int getPublishDataExecutorMinPoolSize(); + + int getPublishDataExecutorMaxPoolSize(); + + int getPublishDataExecutorQueueSize(); + + long getPublishDataExecutorKeepAliveTime(); + + double getAccessLimitRate(); + + int getDataClientConnNum(); + + int getSessionSchedulerPoolSize(); + + boolean isEnableSessionLoadbalancePolicy(); + + int getDefaultRequestExecutorMinPoolSize(); + + int getDefaultRequestExecutorMaxPoolSize(); + + int getDefaultRequestExecutorQueueSize(); + + long getDefaultRequestExecutorKeepAliveTime(); } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfigBean.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfigBean.java index 916c5d641..0bc7c4bd4 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfigBean.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfigBean.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.server.session.bootstrap; -import org.springframework.boot.context.properties.ConfigurationProperties; - -import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang.builder.ToStringStyle; +import org.springframework.boot.context.properties.ConfigurationProperties; + /** * The type Session server config bean. * @author shangyu.wh @@ -48,7 +49,7 @@ public class SessionServerConfigBean implements SessionServerConfig { private int schedulerHeartbeatFirstDelay = 3; - private int schedulerHeartbeatExpBackOffBound = 10; + private int schedulerHeartbeatExpBackOffBound = 1; private int schedulerGetSessionNodeTimeout = 3; @@ -66,19 +67,45 @@ public class SessionServerConfigBean implements SessionServerConfig { private int schedulerConnectMetaFirstDelay = 5; - private int schedulerConnectMetaExpBackOffBound = 10; + private int schedulerConnectMetaExpBackOffBound = 3; + + private int schedulerConnectDataTimeout = 10; + + private int schedulerConnectDataFirstDelay = 10; + + private int schedulerConnectDataExpBackOffBound = 3; + + private int schedulerCleanInvalidClientTimeOut = 3; + + private int schedulerCleanInvalidClientFirstDelay = 10; + + private int schedulerCleanInvalidClientBackOffBound = 5; - private int schedulerConnectDataTimeout = 3; + private int cancelDataTaskRetryTimes = 2; - private int schedulerConnectDataFirstDelay = 3; + private long cancelDataTaskRetryFirstDelay = 500; - private int schedulerConnectDataExpBackOffBound = 10; + private long cancelDataTaskRetryIncrementDelay = 500; - private int cancelDataTaskRetryTimes = 5; + private int publishDataTaskRetryTimes = 2; - private int cancelDataTaskRetryFirstDelay = 100; + private long publishDataTaskRetryFirstDelay = 3000; - private long cancelDataTaskRetryIncrementDelay = 200; + private long publishDataTaskRetryIncrementDelay = 5000; + + private int unPublishDataTaskRetryTimes = 2; + + private long unPublishDataTaskRetryFirstDelay = 3000; + + private long unPublishDataTaskRetryIncrementDelay = 5000; + + private int datumSnapshotTaskRetryTimes = 1; + + private long datumSnapshotTaskRetryFirstDelay = 5000; + + private long datumSnapshotTaskRetryIncrementDelay = 5000; + + private int renewDatumTaskRetryTimes = 1; private int dataChangeFetchTaskRetryTimes = 3; @@ -118,11 +145,11 @@ public class SessionServerConfigBean implements SessionServerConfig { private long dataChangeExecutorKeepAliveTime = 60; - private int disconnectClientExecutorMinPoolSize = 40; + private int connectClientExecutorMinPoolSize = 60; - private int disconnectClientExecutorMaxPoolSize = 200; + private int connectClientExecutorMaxPoolSize = 400; - private int disconnectClientExecutorQueueSize = 10000; + private int connectClientExecutorQueueSize = 10000; private int dataChangeFetchTaskMaxBufferSize = 1000000; @@ -132,6 +159,8 @@ public class SessionServerConfigBean implements SessionServerConfig { private int dataNodeExchangeTimeOut = 3000; + private int dataNodeExchangeForFetchDatumTimeOut = 5000; + private int metaNodeExchangeTimeOut = 3000; private int numberOfReplicas = 1000; @@ -140,17 +169,61 @@ public class SessionServerConfigBean implements SessionServerConfig { private int userDataPushRetryWheelTicksDuration = 100; + private int userDataPushRetryExecutorQueueSize = 1000000; + + private int userDataPushRetryExecutorThreadSize = 10; + + private int renewDatumWheelTicksSize = 2048; + + private int renewDatumWheelTicksDuration = 500; + + private int renewDatumWheelTaskDelaySec = 180; + + private int renewDatumWheelTaskRandomFirstDelaySec = 200; + + private int renewDatumWheelThreadSize = 10; + + private int renewDatumWheelQueueSize = 10000; + private int pushDataTaskRetryFirstDelay = 500; private long pushDataTaskRetryIncrementDelay = 500; + private long pushTaskConfirmWaitTimeout = 10000; + + private int pushTaskConfirmCheckWheelTicksSize = 1024; + + private int pushTaskConfirmCheckWheelTicksDuration = 100; + + private int pushTaskConfirmCheckExecutorQueueSize = 10000; + + private int pushTaskConfirmCheckExecutorThreadSize = 10; + + private int publishDataExecutorMinPoolSize = 100; + + private int publishDataExecutorMaxPoolSize = 400; + + private int publishDataExecutorQueueSize = 10000; + + private long publishDataExecutorKeepAliveTime = 60; + + private int defaultRequestExecutorMinPoolSize = 20; + + private int defaultRequestExecutorMaxPoolSize = 400; + + private int defaultRequestExecutorQueueSize = 600; + + private long defaultRequestExecutorKeepAliveTime = 60; + + private double accessLimitRate = 100000.0; + private String sessionServerRegion; private String sessionServerDataCenter; private boolean stopPushSwitch = false; - private boolean beginDataFetchTask = false; + private boolean beginDataFetchTask = true; //begin config for enterprise version @@ -163,9 +236,23 @@ public class SessionServerConfigBean implements SessionServerConfig { private Pattern invalidIgnoreDataIdPattern = null; - private String pushEmptyDataDataIdPrefixes = ""; + private String blacklistPubDataIdRegex = ""; + + private String blacklistSubDataIdRegex = ""; + + private int renewAndSnapshotSilentPeriodSec = 20; + + private int writeDataAcceptorQueueSize = 10000; + + private int dataNodeRetryExecutorQueueSize = 1000000; + + private int dataNodeRetryExecutorThreadSize = 100; + + private int dataClientConnNum = 10; + + private int sessionSchedulerPoolSize = 6; - private Set pushEmptyDataDataIdPrefixesSet; + private boolean enableSessionLoadbalancePolicy = false; //end config for enterprise version @@ -179,6 +266,230 @@ public SessionServerConfigBean(CommonConfig commonConfig) { this.commonConfig = commonConfig; } + /** + * Getter method for property renewDatumWheelThreadSize. + * + * @return property value of renewDatumWheelThreadSize + */ + public int getRenewDatumWheelThreadSize() { + return renewDatumWheelThreadSize; + } + + /** + * Setter method for property renewDatumWheelThreadSize . + * + * @param renewDatumWheelThreadSize value to be assigned to property renewDatumWheelThreadSize + */ + public void setRenewDatumWheelThreadSize(int renewDatumWheelThreadSize) { + this.renewDatumWheelThreadSize = renewDatumWheelThreadSize; + } + + /** + * Getter method for property renewDatumWheelQueueSize. + * + * @return property value of renewDatumWheelQueueSize + */ + public int getRenewDatumWheelQueueSize() { + return renewDatumWheelQueueSize; + } + + /** + * Setter method for property renewDatumWheelQueueSize . + * + * @param renewDatumWheelQueueSize value to be assigned to property renewDatumWheelQueueSize + */ + public void setRenewDatumWheelQueueSize(int renewDatumWheelQueueSize) { + this.renewDatumWheelQueueSize = renewDatumWheelQueueSize; + } + + /** + * Getter method for property userDataPushRetryExecutorQueueSize. + * + * @return property value of userDataPushRetryExecutorQueueSize + */ + @Override + public int getUserDataPushRetryExecutorQueueSize() { + return userDataPushRetryExecutorQueueSize; + } + + /** + * Setter method for property userDataPushRetryExecutorQueueSize . + * + * @param userDataPushRetryExecutorQueueSize value to be assigned to property userDataPushRetryExecutorQueueSize + */ + public void setUserDataPushRetryExecutorQueueSize(int userDataPushRetryExecutorQueueSize) { + this.userDataPushRetryExecutorQueueSize = userDataPushRetryExecutorQueueSize; + } + + /** + * Getter method for property userDataPushRetryExecutorThreadSize. + * + * @return property value of userDataPushRetryExecutorThreadSize + */ + @Override + public int getUserDataPushRetryExecutorThreadSize() { + return userDataPushRetryExecutorThreadSize; + } + + /** + * Setter method for property userDataPushRetryExecutorThreadSize . + * + * @param userDataPushRetryExecutorThreadSize value to be assigned to property userDataPushRetryExecutorThreadSize + */ + public void setUserDataPushRetryExecutorThreadSize(int userDataPushRetryExecutorThreadSize) { + this.userDataPushRetryExecutorThreadSize = userDataPushRetryExecutorThreadSize; + } + + /** + * Getter method for property dataNodeRetryExecutorThreadSize. + * + * @return property value of dataNodeRetryExecutorThreadSize + */ + @Override + public int getDataNodeRetryExecutorThreadSize() { + return dataNodeRetryExecutorThreadSize; + } + + /** + * Setter method for property dataNodeRetryExecutorThreadSize . + * + * @param dataNodeRetryExecutorThreadSize value to be assigned to property dataNodeRetryExecutorThreadSize + */ + public void setDataNodeRetryExecutorThreadSize(int dataNodeRetryExecutorThreadSize) { + this.dataNodeRetryExecutorThreadSize = dataNodeRetryExecutorThreadSize; + } + + /** + * Getter method for property dataNodeRetryExecutorQueueSize. + * + * @return property value of dataNodeRetryExecutorQueueSize + */ + @Override + public int getDataNodeRetryExecutorQueueSize() { + return dataNodeRetryExecutorQueueSize; + } + + /** + * Setter method for property dataNodeRetryExecutorQueueSize . + * + * @param dataNodeRetryExecutorQueueSize value to be assigned to property dataNodeRetryExecutorQueueSize + */ + public void setDataNodeRetryExecutorQueueSize(int dataNodeRetryExecutorQueueSize) { + this.dataNodeRetryExecutorQueueSize = dataNodeRetryExecutorQueueSize; + } + + /** + * Getter method for property writeDataAcceptorQueueSize. + * + * @return property value of writeDataAcceptorQueueSize + */ + public int getWriteDataAcceptorQueueSize() { + return writeDataAcceptorQueueSize; + } + + /** + * Setter method for property writeDataAcceptorQueueSize . + * + * @param writeDataAcceptorQueueSize value to be assigned to property writeDataAcceptorQueueSize + */ + public void setWriteDataAcceptorQueueSize(int writeDataAcceptorQueueSize) { + this.writeDataAcceptorQueueSize = writeDataAcceptorQueueSize; + } + + /** + * Getter method for property renewAndSnapshotSilentPeriodSec. + * + * @return property value of renewAndSnapshotSilentPeriodSec + */ + public int getRenewAndSnapshotSilentPeriodSec() { + return renewAndSnapshotSilentPeriodSec; + } + + /** + * Setter method for property renewAndSnapshotSilentPeriodSec . + * + * @param renewAndSnapshotSilentPeriodSec value to be assigned to property renewAndSnapshotSilentPeriodSec + */ + public void setRenewAndSnapshotSilentPeriodSec(int renewAndSnapshotSilentPeriodSec) { + this.renewAndSnapshotSilentPeriodSec = renewAndSnapshotSilentPeriodSec; + } + + /** + * Getter method for property publishDataTaskRetryTimes. + * + * @return property value of publishDataTaskRetryTimes + */ + @Override + public int getPublishDataTaskRetryTimes() { + return publishDataTaskRetryTimes; + } + + /** + * Setter method for property publishDataTaskRetryTimes . + * + * @param publishDataTaskRetryTimes value to be assigned to property publishDataTaskRetryTimes + */ + public void setPublishDataTaskRetryTimes(int publishDataTaskRetryTimes) { + this.publishDataTaskRetryTimes = publishDataTaskRetryTimes; + } + + /** + * Getter method for property unPublishDataTaskRetryTimes. + * + * @return property value of unPublishDataTaskRetryTimes + */ + @Override + public int getUnPublishDataTaskRetryTimes() { + return unPublishDataTaskRetryTimes; + } + + /** + * Setter method for property unPublishDataTaskRetryTimes . + * + * @param unPublishDataTaskRetryTimes value to be assigned to property unPublishDataTaskRetryTimes + */ + public void setUnPublishDataTaskRetryTimes(int unPublishDataTaskRetryTimes) { + this.unPublishDataTaskRetryTimes = unPublishDataTaskRetryTimes; + } + + /** + * Getter method for property datumSnapshotTaskRetryTimes. + * + * @return property value of datumSnapshotTaskRetryTimes + */ + @Override + public int getDatumSnapshotTaskRetryTimes() { + return datumSnapshotTaskRetryTimes; + } + + /** + * Setter method for property datumSnapshotTaskRetryTimes . + * + * @param datumSnapshotTaskRetryTimes value to be assigned to property datumSnapshotTaskRetryTimes + */ + public void setDatumSnapshotTaskRetryTimes(int datumSnapshotTaskRetryTimes) { + this.datumSnapshotTaskRetryTimes = datumSnapshotTaskRetryTimes; + } + + /** + * Getter method for property renewDatumTaskRetryTimes. + * + * @return property value of renewDatumTaskRetryTimes + */ + @Override + public int getRenewDatumTaskRetryTimes() { + return renewDatumTaskRetryTimes; + } + + /** + * Setter method for property renewDatumTaskRetryTimes . + * + * @param renewDatumTaskRetryTimes value to be assigned to property renewDatumTaskRetryTimes + */ + public void setRenewDatumTaskRetryTimes(int renewDatumTaskRetryTimes) { + this.renewDatumTaskRetryTimes = renewDatumTaskRetryTimes; + } + /** * Getter method for property serverPort. * @@ -337,19 +648,10 @@ public void setCancelDataTaskRetryTimes(int cancelDataTaskRetryTimes) { * @return property value of cancelDataTaskRetryFirstDelay */ @Override - public int getCancelDataTaskRetryFirstDelay() { + public long getCancelDataTaskRetryFirstDelay() { return cancelDataTaskRetryFirstDelay; } - /** - * Setter method for property cancelDataTaskRetryFirstDelay. - * - * @param cancelDataTaskRetryFirstDelay value to be assigned to property cancelDataTaskRetryFirstDelay - */ - public void setCancelDataTaskRetryFirstDelay(int cancelDataTaskRetryFirstDelay) { - this.cancelDataTaskRetryFirstDelay = cancelDataTaskRetryFirstDelay; - } - /** * Getter method for property cancelDataTaskRetryIncrementDelay. * @@ -369,6 +671,129 @@ public void setCancelDataTaskRetryIncrementDelay(long cancelDataTaskRetryIncreme this.cancelDataTaskRetryIncrementDelay = cancelDataTaskRetryIncrementDelay; } + /** + * Setter method for property cancelDataTaskRetryFirstDelay . + * + * @param cancelDataTaskRetryFirstDelay value to be assigned to property cancelDataTaskRetryFirstDelay + */ + public void setCancelDataTaskRetryFirstDelay(long cancelDataTaskRetryFirstDelay) { + this.cancelDataTaskRetryFirstDelay = cancelDataTaskRetryFirstDelay; + } + + /** + * Getter method for property publishDataTaskRetryFirstDelay. + * + * @return property value of publishDataTaskRetryFirstDelay + */ + @Override + public long getPublishDataTaskRetryFirstDelay() { + return publishDataTaskRetryFirstDelay; + } + + /** + * Setter method for property publishDataTaskRetryFirstDelay . + * + * @param publishDataTaskRetryFirstDelay value to be assigned to property publishDataTaskRetryFirstDelay + */ + public void setPublishDataTaskRetryFirstDelay(long publishDataTaskRetryFirstDelay) { + this.publishDataTaskRetryFirstDelay = publishDataTaskRetryFirstDelay; + } + + /** + * Getter method for property publishDataTaskRetryIncrementDelay. + * + * @return property value of publishDataTaskRetryIncrementDelay + */ + @Override + public long getPublishDataTaskRetryIncrementDelay() { + return publishDataTaskRetryIncrementDelay; + } + + /** + * Setter method for property publishDataTaskRetryIncrementDelay . + * + * @param publishDataTaskRetryIncrementDelay value to be assigned to property publishDataTaskRetryIncrementDelay + */ + public void setPublishDataTaskRetryIncrementDelay(long publishDataTaskRetryIncrementDelay) { + this.publishDataTaskRetryIncrementDelay = publishDataTaskRetryIncrementDelay; + } + + /** + * Getter method for property unPublishDataTaskRetryFirstDelay. + * + * @return property value of unPublishDataTaskRetryFirstDelay + */ + @Override + public long getUnPublishDataTaskRetryFirstDelay() { + return unPublishDataTaskRetryFirstDelay; + } + + /** + * Setter method for property unPublishDataTaskRetryFirstDelay . + * + * @param unPublishDataTaskRetryFirstDelay value to be assigned to property unPublishDataTaskRetryFirstDelay + */ + public void setUnPublishDataTaskRetryFirstDelay(long unPublishDataTaskRetryFirstDelay) { + this.unPublishDataTaskRetryFirstDelay = unPublishDataTaskRetryFirstDelay; + } + + /** + * Getter method for property unPublishDataTaskRetryIncrementDelay. + * + * @return property value of unPublishDataTaskRetryIncrementDelay + */ + @Override + public long getUnPublishDataTaskRetryIncrementDelay() { + return unPublishDataTaskRetryIncrementDelay; + } + + /** + * Setter method for property unPublishDataTaskRetryIncrementDelay . + * + * @param unPublishDataTaskRetryIncrementDelay value to be assigned to property unPublishDataTaskRetryIncrementDelay + */ + public void setUnPublishDataTaskRetryIncrementDelay(long unPublishDataTaskRetryIncrementDelay) { + this.unPublishDataTaskRetryIncrementDelay = unPublishDataTaskRetryIncrementDelay; + } + + /** + * Getter method for property datumSnapshotTaskRetryFirstDelay. + * + * @return property value of datumSnapshotTaskRetryFirstDelay + */ + @Override + public long getDatumSnapshotTaskRetryFirstDelay() { + return datumSnapshotTaskRetryFirstDelay; + } + + /** + * Setter method for property datumSnapshotTaskRetryFirstDelay . + * + * @param datumSnapshotTaskRetryFirstDelay value to be assigned to property datumSnapshotTaskRetryFirstDelay + */ + public void setDatumSnapshotTaskRetryFirstDelay(long datumSnapshotTaskRetryFirstDelay) { + this.datumSnapshotTaskRetryFirstDelay = datumSnapshotTaskRetryFirstDelay; + } + + /** + * Getter method for property datumSnapshotTaskRetryIncrementDelay. + * + * @return property value of datumSnapshotTaskRetryIncrementDelay + */ + @Override + public long getDatumSnapshotTaskRetryIncrementDelay() { + return datumSnapshotTaskRetryIncrementDelay; + } + + /** + * Setter method for property datumSnapshotTaskRetryIncrementDelay . + * + * @param datumSnapshotTaskRetryIncrementDelay value to be assigned to property datumSnapshotTaskRetryIncrementDelay + */ + public void setDatumSnapshotTaskRetryIncrementDelay(long datumSnapshotTaskRetryIncrementDelay) { + this.datumSnapshotTaskRetryIncrementDelay = datumSnapshotTaskRetryIncrementDelay; + } + /** * Getter method for property receivedDataMultiPushTaskRetryTimes. * @@ -517,6 +942,11 @@ public String getSessionServerRegion() { return sessionServerRegion; } + @Override + public String getClientCell(String subscriberCell) { + return this.getSessionServerRegion(); + } + /** * Setter method for property sessionServerRegion. * @@ -754,32 +1184,86 @@ public void setSchedulerConnectDataTimeout(int schedulerConnectDataTimeout) { this.schedulerConnectDataTimeout = schedulerConnectDataTimeout; } - @Override - public int getSchedulerConnectDataFirstDelay() { - return schedulerConnectDataFirstDelay; + @Override + public int getSchedulerConnectDataFirstDelay() { + return schedulerConnectDataFirstDelay; + } + + /** + * Setter method for property schedulerConnectDataFirstDelay. + * + * @param schedulerConnectDataFirstDelay value to be assigned to property schedulerConnectDataFirstDelay + */ + public void setSchedulerConnectDataFirstDelay(int schedulerConnectDataFirstDelay) { + this.schedulerConnectDataFirstDelay = schedulerConnectDataFirstDelay; + } + + @Override + public int getSchedulerConnectDataExpBackOffBound() { + return schedulerConnectDataExpBackOffBound; + } + + /** + * Setter method for property schedulerConnectDataExpBackOffBound. + * + * @param schedulerConnectDataExpBackOffBound value to be assigned to property schedulerConnectDataExpBackOffBound + */ + public void setSchedulerConnectDataExpBackOffBound(int schedulerConnectDataExpBackOffBound) { + this.schedulerConnectDataExpBackOffBound = schedulerConnectDataExpBackOffBound; + } + + /** + * Getter method for property schedulerCleanInvolidClientTimeOut. + * + * @return property value of schedulerCleanInvolidClientTimeOut + */ + public int getSchedulerCleanInvalidClientTimeOut() { + return schedulerCleanInvalidClientTimeOut; + } + + /** + * Getter method for property schedulerCleanInvolidClientFirstDelay. + * + * @return property value of schedulerCleanInvolidClientFirstDelay + */ + public int getSchedulerCleanInvalidClientFirstDelay() { + return schedulerCleanInvalidClientFirstDelay; + } + + /** + * Getter method for property schedulerCleanInvolidClientBackOffBound. + * + * @return property value of schedulerCleanInvolidClientBackOffBound + */ + public int getSchedulerCleanInvalidClientBackOffBound() { + return schedulerCleanInvalidClientBackOffBound; } /** - * Setter method for property schedulerConnectDataFirstDelay. + * Setter method for property schedulerCleanInvolidClientTimeOut. * - * @param schedulerConnectDataFirstDelay value to be assigned to property schedulerConnectDataFirstDelay + * @param schedulerCleanInvalidClientTimeOut value to be assigned to property schedulerCleanInvolidClientTimeOut */ - public void setSchedulerConnectDataFirstDelay(int schedulerConnectDataFirstDelay) { - this.schedulerConnectDataFirstDelay = schedulerConnectDataFirstDelay; + public void setSchedulerCleanInvalidClientTimeOut(int schedulerCleanInvalidClientTimeOut) { + this.schedulerCleanInvalidClientTimeOut = schedulerCleanInvalidClientTimeOut; } - @Override - public int getSchedulerConnectDataExpBackOffBound() { - return schedulerConnectDataExpBackOffBound; + /** + * Setter method for property schedulerCleanInvolidClientFirstDelay. + * + * @param schedulerCleanInvalidClientFirstDelay value to be assigned to property schedulerCleanInvolidClientFirstDelay + */ + public void setSchedulerCleanInvalidClientFirstDelay(int schedulerCleanInvalidClientFirstDelay) { + this.schedulerCleanInvalidClientFirstDelay = schedulerCleanInvalidClientFirstDelay; } /** - * Setter method for property schedulerConnectDataExpBackOffBound. + * Setter method for property schedulerCleanInvolidClientBackOffBound. * - * @param schedulerConnectDataExpBackOffBound value to be assigned to property schedulerConnectDataExpBackOffBound + * @param schedulerCleanInvalidClientBackOffBound value to be assigned to property schedulerCleanInvolidClientBackOffBound */ - public void setSchedulerConnectDataExpBackOffBound(int schedulerConnectDataExpBackOffBound) { - this.schedulerConnectDataExpBackOffBound = schedulerConnectDataExpBackOffBound; + public void setSchedulerCleanInvalidClientBackOffBound(int schedulerCleanInvalidClientBackOffBound) { + this.schedulerCleanInvalidClientBackOffBound = schedulerCleanInvalidClientBackOffBound; } /** @@ -904,10 +1388,6 @@ public void setAccessDataExecutorKeepAliveTime(long accessDataExecutorKeepAliveT this.accessDataExecutorKeepAliveTime = accessDataExecutorKeepAliveTime; } - public String getPushEmptyDataDataIdPrefixes() { - return pushEmptyDataDataIdPrefixes; - } - /** * Getter method for property dataChangeExecutorMinPoolSize. * @@ -984,15 +1464,6 @@ public void setDataChangeExecutorKeepAliveTime(long dataChangeExecutorKeepAliveT this.dataChangeExecutorKeepAliveTime = dataChangeExecutorKeepAliveTime; } - /** - * Setter method for property pushEmptyDataDataIdPrefixes. - * - * @param pushEmptyDataDataIdPrefixes value to be assigned to property pushEmptyDataDataIdPrefixes - */ - public void setPushEmptyDataDataIdPrefixes(String pushEmptyDataDataIdPrefixes) { - this.pushEmptyDataDataIdPrefixes = pushEmptyDataDataIdPrefixes; - } - /** * Getter method for property pushTaskExecutorMinPoolSize. * @@ -1069,29 +1540,6 @@ public void setPushTaskExecutorKeepAliveTime(long pushTaskExecutorKeepAliveTime) this.pushTaskExecutorKeepAliveTime = pushTaskExecutorKeepAliveTime; } - public Set getPushEmptyDataDataIdPrefixesSet() { - if (pushEmptyDataDataIdPrefixesSet == null || pushEmptyDataDataIdPrefixesSet.isEmpty()) { - Set s = new HashSet<>(); - String[] arr = pushEmptyDataDataIdPrefixes.split(";"); - for (String str : arr) { - if (str.trim().length() > 0) { - s.add(str); - } - } - pushEmptyDataDataIdPrefixesSet = Collections.unmodifiableSet(s); - } - return pushEmptyDataDataIdPrefixesSet; - } - - /** - * Setter method for property pushEmptyDataDataIdPrefixesSet. - * - * @param pushEmptyDataDataIdPrefixesSet value to be assigned to property pushEmptyDataDataIdPrefixesSet - */ - public void setPushEmptyDataDataIdPrefixesSet(Set pushEmptyDataDataIdPrefixesSet) { - this.pushEmptyDataDataIdPrefixesSet = pushEmptyDataDataIdPrefixesSet; - } - /** * Getter method for property defaultSessionExecutorMinPoolSize. * @@ -1147,60 +1595,57 @@ public void setDefaultSessionExecutorKeepAliveTime(long defaultSessionExecutorKe } /** - * Getter method for property disconnectClientExecutorMinPoolSize. + * Getter method for property connectClientExecutorMinPoolSize. * - * @return property value of disconnectClientExecutorMinPoolSize + * @return property value of connectClientExecutorMinPoolSize */ - @Override - public int getDisconnectClientExecutorMinPoolSize() { - return disconnectClientExecutorMinPoolSize; + public int getConnectClientExecutorMinPoolSize() { + return connectClientExecutorMinPoolSize; } /** - * Setter method for property disconnectClientExecutorMinPoolSize. + * Getter method for property connectClientExecutorMaxPoolSize. * - * @param disconnectClientExecutorMinPoolSize value to be assigned to property disconnectClientExecutorMinPoolSize + * @return property value of connectClientExecutorMaxPoolSize */ - public void setDisconnectClientExecutorMinPoolSize(int disconnectClientExecutorMinPoolSize) { - this.disconnectClientExecutorMinPoolSize = disconnectClientExecutorMinPoolSize; + public int getConnectClientExecutorMaxPoolSize() { + return connectClientExecutorMaxPoolSize; } /** - * Getter method for property disconnectClientExecutorMaxPoolSize. + * Getter method for property connectClientExecutorQueueSize. * - * @return property value of disconnectClientExecutorMaxPoolSize + * @return property value of connectClientExecutorQueueSize */ - @Override - public int getDisconnectClientExecutorMaxPoolSize() { - return disconnectClientExecutorMaxPoolSize; + public int getConnectClientExecutorQueueSize() { + return connectClientExecutorQueueSize; } /** - * Setter method for property disconnectClientExecutorMaxPoolSize. + * Setter method for property connectClientExecutorMinPoolSize. * - * @param disconnectClientExecutorMaxPoolSize value to be assigned to property disconnectClientExecutorMaxPoolSize + * @param connectClientExecutorMinPoolSize value to be assigned to property connectClientExecutorMinPoolSize */ - public void setDisconnectClientExecutorMaxPoolSize(int disconnectClientExecutorMaxPoolSize) { - this.disconnectClientExecutorMaxPoolSize = disconnectClientExecutorMaxPoolSize; + public void setConnectClientExecutorMinPoolSize(int connectClientExecutorMinPoolSize) { + this.connectClientExecutorMinPoolSize = connectClientExecutorMinPoolSize; } /** - * Getter method for property disconnectClientExecutorQueueSize. + * Setter method for property connectClientExecutorMaxPoolSize. * - * @return property value of disconnectClientExecutorQueueSize + * @param connectClientExecutorMaxPoolSize value to be assigned to property connectClientExecutorMaxPoolSize */ - @Override - public int getDisconnectClientExecutorQueueSize() { - return disconnectClientExecutorQueueSize; + public void setConnectClientExecutorMaxPoolSize(int connectClientExecutorMaxPoolSize) { + this.connectClientExecutorMaxPoolSize = connectClientExecutorMaxPoolSize; } /** - * Setter method for property disconnectClientExecutorQueueSize. + * Setter method for property connectClientExecutorQueueSize. * - * @param disconnectClientExecutorQueueSize value to be assigned to property disconnectClientExecutorQueueSize + * @param connectClientExecutorQueueSize value to be assigned to property connectClientExecutorQueueSize */ - public void setDisconnectClientExecutorQueueSize(int disconnectClientExecutorQueueSize) { - this.disconnectClientExecutorQueueSize = disconnectClientExecutorQueueSize; + public void setConnectClientExecutorQueueSize(int connectClientExecutorQueueSize) { + this.connectClientExecutorQueueSize = connectClientExecutorQueueSize; } /** @@ -1308,6 +1753,78 @@ public long getPushDataTaskRetryIncrementDelay() { return pushDataTaskRetryIncrementDelay; } + /** + * Getter method for property renewDatumWheelTicksSize. + * + * @return property value of renewDatumWheelTicksSize + */ + public int getRenewDatumWheelTicksSize() { + return renewDatumWheelTicksSize; + } + + /** + * Getter method for property renewDatumWheelTicksDuration. + * + * @return property value of renewDatumWheelTicksDuration + */ + public int getRenewDatumWheelTicksDuration() { + return renewDatumWheelTicksDuration; + } + + /** + * Getter method for property renewDatumWheelTaskDelaySec. + * + * @return property value of renewDatumWheelTaskDelaySec + */ + public int getRenewDatumWheelTaskDelaySec() { + return renewDatumWheelTaskDelaySec; + } + + /** + * Setter method for property renewDatumWheelTaskDelaySec . + * + * @param renewDatumWheelTaskDelaySec value to be assigned to property renewDatumWheelTaskDelaySec + */ + public void setRenewDatumWheelTaskDelaySec(int renewDatumWheelTaskDelaySec) { + this.renewDatumWheelTaskDelaySec = renewDatumWheelTaskDelaySec; + } + + /** + * Getter method for property renewDatumWheelTaskRandomFirstDelaySec. + * + * @return property value of renewDatumWheelTaskRandomFirstDelaySec + */ + public int getRenewDatumWheelTaskRandomFirstDelaySec() { + return renewDatumWheelTaskRandomFirstDelaySec; + } + + /** + * Setter method for property renewDatumWheelTaskRandomFirstDelaySec . + * + * @param renewDatumWheelTaskRandomFirstDelaySec value to be assigned to property renewDatumWheelTaskRandomFirstDelaySec + */ + public void setRenewDatumWheelTaskRandomFirstDelaySec(int renewDatumWheelTaskRandomFirstDelaySec) { + this.renewDatumWheelTaskRandomFirstDelaySec = renewDatumWheelTaskRandomFirstDelaySec; + } + + /** + * Setter method for property renewDatumWheelTicksSize. + * + * @param renewDatumWheelTicksSize value to be assigned to property renewDatumWheelTicksSize + */ + public void setRenewDatumWheelTicksSize(int renewDatumWheelTicksSize) { + this.renewDatumWheelTicksSize = renewDatumWheelTicksSize; + } + + /** + * Setter method for property renewDatumWheelTicksDuration. + * + * @param renewDatumWheelTicksDuration value to be assigned to property renewDatumWheelTicksDuration + */ + public void setRenewDatumWheelTicksDuration(int renewDatumWheelTicksDuration) { + this.renewDatumWheelTicksDuration = renewDatumWheelTicksDuration; + } + /** * Setter method for property pushDataTaskRetryIncrementDelay. * @@ -1317,6 +1834,241 @@ public void setPushDataTaskRetryIncrementDelay(long pushDataTaskRetryIncrementDe this.pushDataTaskRetryIncrementDelay = pushDataTaskRetryIncrementDelay; } + /** + * Getter method for property blacklistPubDataIdRegex. + * + * @return property value of blacklistPubDataIdRegex + */ + public String getBlacklistPubDataIdRegex() { + return blacklistPubDataIdRegex; + } + + /** + * Getter method for property blacklistSubDataIdRegex. + * + * @return property value of blacklistSubDataIdRegex + */ + public String getBlacklistSubDataIdRegex() { + return blacklistSubDataIdRegex; + } + + /** + * Setter method for property blacklistPubDataIdRegex. + * + * @param blacklistPubDataIdRegex value to be assigned to property blacklistPubDataIdRegex + */ + public void setBlacklistPubDataIdRegex(String blacklistPubDataIdRegex) { + this.blacklistPubDataIdRegex = blacklistPubDataIdRegex; + } + + /** + * Setter method for property blacklistSubDataIdRegex. + * + * @param blacklistSubDataIdRegex value to be assigned to property blacklistSubDataIdRegex + */ + public void setBlacklistSubDataIdRegex(String blacklistSubDataIdRegex) { + this.blacklistSubDataIdRegex = blacklistSubDataIdRegex; + } + + /** + * Getter method for property pushTaskConfirmWaitTimeout. + * + * @return property value of pushTaskConfirmWaitTimeout + */ + public long getPushTaskConfirmWaitTimeout() { + return pushTaskConfirmWaitTimeout; + } + + /** + * Setter method for property pushTaskConfirmWaitTimeout. + * + * @param pushTaskConfirmWaitTimeout value to be assigned to property pushTaskConfirmWaitTimeout + */ + public void setPushTaskConfirmWaitTimeout(long pushTaskConfirmWaitTimeout) { + this.pushTaskConfirmWaitTimeout = pushTaskConfirmWaitTimeout; + } + + /** + * Getter method for property pushTaskConfirmCheckWheelTicksSize. + * + * @return property value of pushTaskConfirmCheckWheelTicksSize + */ + public int getPushTaskConfirmCheckWheelTicksSize() { + return pushTaskConfirmCheckWheelTicksSize; + } + + /** + * Getter method for property pushTaskConfirmCheckWheelTicksDuration. + * + * @return property value of pushTaskConfirmCheckWheelTicksDuration + */ + public int getPushTaskConfirmCheckWheelTicksDuration() { + return pushTaskConfirmCheckWheelTicksDuration; + } + + /** + * Getter method for property pushTaskConfirmCheckExecutorQueueSize. + * + * @return property value of pushTaskConfirmCheckExecutorQueueSize + */ + public int getPushTaskConfirmCheckExecutorQueueSize() { + return pushTaskConfirmCheckExecutorQueueSize; + } + + /** + * Getter method for property pushTaskConfirmCheckExecutorThreadSize. + * + * @return property value of pushTaskConfirmCheckExecutorThreadSize + */ + public int getPushTaskConfirmCheckExecutorThreadSize() { + return pushTaskConfirmCheckExecutorThreadSize; + } + + /** + * Setter method for property pushTaskConfirmCheckWheelTicksSize. + * + * @param pushTaskConfirmCheckWheelTicksSize value to be assigned to property pushTaskConfirmCheckWheelTicksSize + */ + public void setPushTaskConfirmCheckWheelTicksSize(int pushTaskConfirmCheckWheelTicksSize) { + this.pushTaskConfirmCheckWheelTicksSize = pushTaskConfirmCheckWheelTicksSize; + } + + /** + * Setter method for property pushTaskConfirmCheckWheelTicksDuration. + * + * @param pushTaskConfirmCheckWheelTicksDuration value to be assigned to property pushTaskConfirmCheckWheelTicksDuration + */ + public void setPushTaskConfirmCheckWheelTicksDuration(int pushTaskConfirmCheckWheelTicksDuration) { + this.pushTaskConfirmCheckWheelTicksDuration = pushTaskConfirmCheckWheelTicksDuration; + } + + /** + * Setter method for property pushTaskConfirmCheckExecutorQueueSize. + * + * @param pushTaskConfirmCheckExecutorQueueSize value to be assigned to property pushTaskConfirmCheckExecutorQueueSize + */ + public void setPushTaskConfirmCheckExecutorQueueSize(int pushTaskConfirmCheckExecutorQueueSize) { + this.pushTaskConfirmCheckExecutorQueueSize = pushTaskConfirmCheckExecutorQueueSize; + } + + /** + * Setter method for property pushTaskConfirmCheckExecutorThreadSize. + * + * @param pushTaskConfirmCheckExecutorThreadSize value to be assigned to property pushTaskConfirmCheckExecutorThreadSize + */ + public void setPushTaskConfirmCheckExecutorThreadSize(int pushTaskConfirmCheckExecutorThreadSize) { + this.pushTaskConfirmCheckExecutorThreadSize = pushTaskConfirmCheckExecutorThreadSize; + } + + /** + * Getter method for property publishDataExecutorMinPoolSize. + * + * @return property value of publishDataExecutorMinPoolSize + */ + public int getPublishDataExecutorMinPoolSize() { + return publishDataExecutorMinPoolSize; + } + + /** + * Getter method for property publishDataExecutorMaxPoolSize. + * + * @return property value of publishDataExecutorMaxPoolSize + */ + public int getPublishDataExecutorMaxPoolSize() { + return publishDataExecutorMaxPoolSize; + } + + /** + * Getter method for property publishDataExecutorQueueSize. + * + * @return property value of publishDataExecutorQueueSize + */ + public int getPublishDataExecutorQueueSize() { + return publishDataExecutorQueueSize; + } + + /** + * Getter method for property publishDataExecutorKeepAliveTime. + * + * @return property value of publishDataExecutorKeepAliveTime + */ + public long getPublishDataExecutorKeepAliveTime() { + return publishDataExecutorKeepAliveTime; + } + + /** + * Setter method for property publishDataExecutorMinPoolSize. + * + * @param publishDataExecutorMinPoolSize value to be assigned to property publishDataExecutorMinPoolSize + */ + public void setPublishDataExecutorMinPoolSize(int publishDataExecutorMinPoolSize) { + this.publishDataExecutorMinPoolSize = publishDataExecutorMinPoolSize; + } + + /** + * Setter method for property publishDataExecutorMaxPoolSize. + * + * @param publishDataExecutorMaxPoolSize value to be assigned to property publishDataExecutorMaxPoolSize + */ + public void setPublishDataExecutorMaxPoolSize(int publishDataExecutorMaxPoolSize) { + this.publishDataExecutorMaxPoolSize = publishDataExecutorMaxPoolSize; + } + + /** + * Setter method for property publishDataExecutorQueueSize. + * + * @param publishDataExecutorQueueSize value to be assigned to property publishDataExecutorQueueSize + */ + public void setPublishDataExecutorQueueSize(int publishDataExecutorQueueSize) { + this.publishDataExecutorQueueSize = publishDataExecutorQueueSize; + } + + /** + * Setter method for property publishDataExecutorKeepAliveTime. + * + * @param publishDataExecutorKeepAliveTime value to be assigned to property publishDataExecutorKeepAliveTime + */ + public void setPublishDataExecutorKeepAliveTime(long publishDataExecutorKeepAliveTime) { + this.publishDataExecutorKeepAliveTime = publishDataExecutorKeepAliveTime; + } + + /** + * Getter method for property accessLimitRate. + * + * @return property value of accessLimitRate + */ + public double getAccessLimitRate() { + return accessLimitRate; + } + + /** + * Setter method for property accessLimitRate. + * + * @param accessLimitRate value to be assigned to property accessLimitRate + */ + public void setAccessLimitRate(double accessLimitRate) { + this.accessLimitRate = accessLimitRate; + } + + /** + * Getter method for property dataClientConnNum. + * + * @return property value of dataClientConnNum + */ + @Override + public int getDataClientConnNum() { + return dataClientConnNum; + } + + /** + * Setter method for property dataClientConnNum . + * + * @param dataClientConnNum value to be assigned to property dataClientConnNum + */ + public void setDataClientConnNum(int dataClientConnNum) { + this.dataClientConnNum = dataClientConnNum; + } + @Override public boolean isInvalidForeverZone(String zoneId) { @@ -1345,7 +2097,89 @@ public boolean isInvalidIgnored(String dataId) { && invalidIgnoreDataIdPattern.matcher(dataId).find(); } + /** + * Getter method for property sessionSchedulerPoolSize. + * + * @return property value of sessionSchedulerPoolSize + */ + @Override + public int getSessionSchedulerPoolSize() { + return sessionSchedulerPoolSize; + } + + /** + * Setter method for property sessionSchedulerPoolSize . + * + * @param sessionSchedulerPoolSize value to be assigned to property sessionSchedulerPoolSize + */ + public void setSessionSchedulerPoolSize(int sessionSchedulerPoolSize) { + this.sessionSchedulerPoolSize = sessionSchedulerPoolSize; + } + + /** + * Getter method for property dataNodeExchangeForFetchDatumTimeOut. + * + * @return property value of dataNodeExchangeForFetchDatumTimeOut + */ + public int getDataNodeExchangeForFetchDatumTimeOut() { + return dataNodeExchangeForFetchDatumTimeOut; + } + + /** + * Setter method for property dataNodeExchangeForFetchDatumTimeOut . + * + * @param dataNodeExchangeForFetchDatumTimeOut value to be assigned to property dataNodeExchangeForFetchDatumTimeOut + */ + public void setDataNodeExchangeForFetchDatumTimeOut(int dataNodeExchangeForFetchDatumTimeOut) { + this.dataNodeExchangeForFetchDatumTimeOut = dataNodeExchangeForFetchDatumTimeOut; + } + public static int cpus() { return Runtime.getRuntime().availableProcessors(); } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } + + public boolean isEnableSessionLoadbalancePolicy() { + return enableSessionLoadbalancePolicy; + } + + public void setEnableSessionLoadbalancePolicy(boolean enableSessionLoadbalancePolicy) { + this.enableSessionLoadbalancePolicy = enableSessionLoadbalancePolicy; + } + + public int getDefaultRequestExecutorMinPoolSize() { + return defaultRequestExecutorMinPoolSize; + } + + public void setDefaultRequestExecutorMinPoolSize(int defaultRequestExecutorMinPoolSize) { + this.defaultRequestExecutorMinPoolSize = defaultRequestExecutorMinPoolSize; + } + + public int getDefaultRequestExecutorMaxPoolSize() { + return defaultRequestExecutorMaxPoolSize; + } + + public void setDefaultRequestExecutorMaxPoolSize(int defaultRequestExecutorMaxPoolSize) { + this.defaultRequestExecutorMaxPoolSize = defaultRequestExecutorMaxPoolSize; + } + + public int getDefaultRequestExecutorQueueSize() { + return defaultRequestExecutorQueueSize; + } + + public void setDefaultRequestExecutorQueueSize(int defaultRequestExecutorQueueSize) { + this.defaultRequestExecutorQueueSize = defaultRequestExecutorQueueSize; + } + + public long getDefaultRequestExecutorKeepAliveTime() { + return defaultRequestExecutorKeepAliveTime; + } + + public void setDefaultRequestExecutorKeepAliveTime(long defaultRequestExecutorKeepAliveTime) { + this.defaultRequestExecutorKeepAliveTime = defaultRequestExecutorKeepAliveTime; + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java index 63dff75e1..9cea1ccd7 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java @@ -16,24 +16,55 @@ */ package com.alipay.sofa.registry.server.session.bootstrap; +import java.util.ArrayList; +import java.util.Collection; + +import com.alipay.sofa.registry.server.session.connections.ConnectionsService; +import com.alipay.sofa.registry.server.session.remoting.handler.*; +import com.alipay.sofa.registry.server.session.resource.*; +import org.glassfish.jersey.jackson.JacksonFeature; +import org.glassfish.jersey.server.ResourceConfig; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + import com.alipay.sofa.registry.remoting.bolt.exchange.BoltExchange; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; import com.alipay.sofa.registry.remoting.jersey.exchange.JerseyExchange; +import com.alipay.sofa.registry.server.session.acceptor.WriteDataAcceptor; +import com.alipay.sofa.registry.server.session.acceptor.WriteDataAcceptorImpl; import com.alipay.sofa.registry.server.session.cache.CacheGenerator; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.cache.DatumCacheGenerator; import com.alipay.sofa.registry.server.session.cache.SessionCacheService; +import com.alipay.sofa.registry.server.session.filter.DataIdMatchStrategy; +import com.alipay.sofa.registry.server.session.filter.IPMatchStrategy; +import com.alipay.sofa.registry.server.session.filter.ProcessFilter; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistManager; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistManagerImpl; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistMatchProcessFilter; +import com.alipay.sofa.registry.server.session.filter.blacklist.DefaultDataIdMatchStrategy; +import com.alipay.sofa.registry.server.session.filter.blacklist.DefaultIPMatchStrategy; +import com.alipay.sofa.registry.server.session.limit.AccessLimitService; +import com.alipay.sofa.registry.server.session.limit.AccessLimitServiceImpl; import com.alipay.sofa.registry.server.session.listener.CancelDataTaskListener; import com.alipay.sofa.registry.server.session.listener.DataChangeFetchCloudTaskListener; import com.alipay.sofa.registry.server.session.listener.DataChangeFetchTaskListener; import com.alipay.sofa.registry.server.session.listener.DataPushTaskListener; +import com.alipay.sofa.registry.server.session.listener.DatumSnapshotTaskListener; import com.alipay.sofa.registry.server.session.listener.ProvideDataChangeFetchTaskListener; +import com.alipay.sofa.registry.server.session.listener.PublishDataTaskListener; import com.alipay.sofa.registry.server.session.listener.ReceivedConfigDataPushTaskListener; import com.alipay.sofa.registry.server.session.listener.ReceivedDataMultiPushTaskListener; +import com.alipay.sofa.registry.server.session.listener.RenewDatumTaskListener; import com.alipay.sofa.registry.server.session.listener.SessionRegisterDataTaskListener; import com.alipay.sofa.registry.server.session.listener.SubscriberMultiFetchTaskListener; +import com.alipay.sofa.registry.server.session.listener.SubscriberPushEmptyTaskListener; import com.alipay.sofa.registry.server.session.listener.SubscriberRegisterFetchTaskListener; +import com.alipay.sofa.registry.server.session.listener.UnPublishDataTaskListener; import com.alipay.sofa.registry.server.session.listener.WatcherRegisterFetchTaskListener; import com.alipay.sofa.registry.server.session.node.DataNodeManager; import com.alipay.sofa.registry.server.session.node.MetaNodeManager; @@ -51,29 +82,18 @@ import com.alipay.sofa.registry.server.session.node.service.DataNodeServiceImpl; import com.alipay.sofa.registry.server.session.node.service.MetaNodeService; import com.alipay.sofa.registry.server.session.node.service.MetaNodeServiceImpl; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessorManager; +import com.alipay.sofa.registry.server.session.provideData.processor.BlackListProvideDataProcessor; +import com.alipay.sofa.registry.server.session.provideData.processor.RenewSnapshotProvideDataProcessor; +import com.alipay.sofa.registry.server.session.provideData.processor.StopPushProvideDataProcessor; import com.alipay.sofa.registry.server.session.registry.Registry; import com.alipay.sofa.registry.server.session.registry.SessionRegistry; import com.alipay.sofa.registry.server.session.remoting.ClientNodeExchanger; import com.alipay.sofa.registry.server.session.remoting.DataNodeExchanger; import com.alipay.sofa.registry.server.session.remoting.MetaNodeExchanger; -import com.alipay.sofa.registry.server.session.remoting.handler.AbstractClientHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.AbstractServerHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.CancelAddressRequestHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.ClientNodeConnectionHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.DataChangeRequestHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.DataNodeConnectionHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.DataPushRequestHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.MetaNodeConnectionHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.NodeChangeResultHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.NotifyProvideDataChangeHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.PublisherHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.SubscriberHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.SyncConfigHandler; -import com.alipay.sofa.registry.server.session.remoting.handler.WatcherHandler; -import com.alipay.sofa.registry.server.session.resource.ClientsOpenResource; -import com.alipay.sofa.registry.server.session.resource.HealthResource; -import com.alipay.sofa.registry.server.session.resource.SessionDigestResource; -import com.alipay.sofa.registry.server.session.resource.SessionOpenResource; +import com.alipay.sofa.registry.server.session.renew.DefaultRenewService; +import com.alipay.sofa.registry.server.session.renew.RenewService; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.scheduler.timertask.SyncClientsHeartbeatTask; import com.alipay.sofa.registry.server.session.store.DataStore; @@ -104,21 +124,16 @@ import com.alipay.sofa.registry.server.session.strategy.impl.DefaultSubscriberRegisterFetchTaskStrategy; import com.alipay.sofa.registry.server.session.strategy.impl.DefaultSyncConfigHandlerStrategy; import com.alipay.sofa.registry.server.session.strategy.impl.DefaultWatcherHandlerStrategy; +import com.alipay.sofa.registry.server.session.wrapper.AccessLimitWrapperInterceptor; +import com.alipay.sofa.registry.server.session.wrapper.BlacklistWrapperInterceptor; +import com.alipay.sofa.registry.server.session.wrapper.ClientCheckWrapperInterceptor; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInterceptor; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInterceptorManager; import com.alipay.sofa.registry.task.batcher.TaskProcessor; import com.alipay.sofa.registry.task.listener.DefaultTaskListenerManager; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; import com.alipay.sofa.registry.util.PropertySplitter; -import org.glassfish.jersey.jackson.JacksonFeature; -import org.glassfish.jersey.server.ResourceConfig; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; - -import java.util.ArrayList; -import java.util.Collection; /** * @@ -144,7 +159,7 @@ public CommonConfig commonConfig() { } @Bean - @ConditionalOnMissingBean(name = "sessionServerConfig") + @ConditionalOnMissingBean public SessionServerConfig sessionServerConfig(CommonConfig commonConfig) { return new SessionServerConfigBean(commonConfig); } @@ -242,6 +257,8 @@ public Collection metaClientHandlers() { list.add(metaNodeConnectionHandler()); list.add(nodeChangeResultHandler()); list.add(notifyProvideDataChangeHandler()); + list.add(loadbalanceMetricsHandler()); + list.add(configureLoadbalanceHandler()); return list; } @@ -274,6 +291,16 @@ public AbstractClientHandler nodeChangeResultHandler() { public AbstractClientHandler notifyProvideDataChangeHandler() { return new NotifyProvideDataChangeHandler(); } + + @Bean + public AbstractClientHandler loadbalanceMetricsHandler() { + return new LoadbalanceMetricsHandler(); + } + + @Bean + public AbstractClientHandler configureLoadbalanceHandler() { + return new ConfigureLoadbalanceHandler(); + } } @Configuration @@ -307,6 +334,11 @@ public HealthResource healthResource() { public ClientsOpenResource clientsOpenResource() { return new ClientsOpenResource(); } + + @Bean + public ConnectionsResource connectionsResource() { + return new ConnectionsResource(); + } } @Configuration @@ -502,12 +534,49 @@ public TaskListener receivedConfigDataPushTaskListener(TaskListenerManager taskL } @Bean + @ConditionalOnMissingBean(name = "cancelDataTaskListener") public TaskListener cancelDataTaskListener(TaskListenerManager taskListenerManager) { TaskListener taskListener = new CancelDataTaskListener(); taskListenerManager.addTaskListener(taskListener); return taskListener; } + @Bean + public TaskListener datumSnapshotTaskListener(TaskListenerManager taskListenerManager) { + TaskListener taskListener = new DatumSnapshotTaskListener(); + taskListenerManager.addTaskListener(taskListener); + return taskListener; + } + + @Bean + public TaskListener publishDataTaskListener(TaskListenerManager taskListenerManager) { + TaskListener taskListener = new PublishDataTaskListener(); + taskListenerManager.addTaskListener(taskListener); + return taskListener; + } + + @Bean + public TaskListener renewDatumTaskListener(TaskListenerManager taskListenerManager) { + TaskListener taskListener = new RenewDatumTaskListener(); + taskListenerManager.addTaskListener(taskListener); + return taskListener; + } + + @Bean + public TaskListener unPublishDataTaskListener(TaskListenerManager taskListenerManager) { + TaskListener taskListener = new UnPublishDataTaskListener(); + taskListenerManager.addTaskListener(taskListener); + return taskListener; + } + + @Bean + @ConditionalOnMissingBean(name = "subscriberPushEmptyTaskListener") + public TaskListener subscriberPushEmptyTaskListener(TaskListenerManager taskListenerManager) { + TaskListener taskListener = new SubscriberPushEmptyTaskListener(); + taskListenerManager.addTaskListener(taskListener); + return taskListener; + } + @Bean public TaskListenerManager taskListenerManager() { return new DefaultTaskListenerManager(); @@ -602,4 +671,119 @@ public ReceivedConfigDataPushTaskStrategy receivedConfigDataPushTaskStrategy() { return new DefaultReceivedConfigDataPushTaskStrategy(); } } + + @Configuration + public static class AccessLimitServiceConfiguration { + @Bean + public AccessLimitService accessLimitService(SessionServerConfig sessionServerConfig) { + return new AccessLimitServiceImpl(sessionServerConfig); + } + } + + @Configuration + public static class SessionFilterConfiguration { + + @Bean + public IPMatchStrategy ipMatchStrategy() { + return new DefaultIPMatchStrategy(); + } + + @Bean + public DataIdMatchStrategy dataIdMatchStrategy() { + return new DefaultDataIdMatchStrategy(); + } + + @Bean + @ConditionalOnMissingBean + public ProcessFilter processFilter() { + return new BlacklistMatchProcessFilter(); + } + + @Bean + public BlacklistManager blacklistManager() { + return new BlacklistManagerImpl(); + } + + @Bean + public WrapperInterceptorManager wrapperInterceptorManager() { + return new WrapperInterceptorManager(); + } + + @Bean + public WrapperInterceptor clientCheckWrapperInterceptor(WrapperInterceptorManager wrapperInterceptorManager) { + ClientCheckWrapperInterceptor clientCheckWrapperInterceptor = new ClientCheckWrapperInterceptor(); + wrapperInterceptorManager.addInterceptor(clientCheckWrapperInterceptor); + return clientCheckWrapperInterceptor; + } + + @Bean + public WrapperInterceptor blacklistWrapperInterceptor(WrapperInterceptorManager wrapperInterceptorManager) { + BlacklistWrapperInterceptor blacklistWrapperInterceptor = new BlacklistWrapperInterceptor(); + wrapperInterceptorManager.addInterceptor(blacklistWrapperInterceptor); + return blacklistWrapperInterceptor; + } + + @Bean + public WrapperInterceptor accessLimitWrapperInterceptor(WrapperInterceptorManager wrapperInterceptorManager) { + AccessLimitWrapperInterceptor accessLimitWrapperInterceptor = new AccessLimitWrapperInterceptor(); + wrapperInterceptorManager.addInterceptor(accessLimitWrapperInterceptor); + return accessLimitWrapperInterceptor; + } + + } + + @Configuration + public static class SessionRenewDatumConfiguration { + + @Bean + public WriteDataAcceptor writeDataAcceptor() { + return new WriteDataAcceptorImpl(); + } + + @Bean + public RenewService renewService() { + return new DefaultRenewService(); + } + } + + @Configuration + public static class SessionConnectionsConfiguration { + @Bean + public ConnectionsService connectionsService() { + return new ConnectionsService(); + } + } + + @Configuration + public static class SessionProvideDataConfiguration { + + @Bean + public ProvideDataProcessor provideDataProcessorManager() { + return new ProvideDataProcessorManager(); + } + + @Bean + public ProvideDataProcessor blackListProvideDataProcessor(ProvideDataProcessor provideDataProcessorManager) { + ProvideDataProcessor blackListProvideDataProcessor = new BlackListProvideDataProcessor(); + ((ProvideDataProcessorManager) provideDataProcessorManager) + .addProvideDataProcessor(blackListProvideDataProcessor); + return blackListProvideDataProcessor; + } + + @Bean + public ProvideDataProcessor renewSnapshotProvideDataProcessor(ProvideDataProcessor provideDataProcessorManager) { + ProvideDataProcessor renewSnapshotProvideDataProcessor = new RenewSnapshotProvideDataProcessor(); + ((ProvideDataProcessorManager) provideDataProcessorManager) + .addProvideDataProcessor(renewSnapshotProvideDataProcessor); + return renewSnapshotProvideDataProcessor; + } + + @Bean + public ProvideDataProcessor stopPushProvideDataProcessor(ProvideDataProcessor provideDataProcessorManager) { + ProvideDataProcessor stopPushProvideDataProcessor = new StopPushProvideDataProcessor(); + ((ProvideDataProcessorManager) provideDataProcessorManager) + .addProvideDataProcessor(stopPushProvideDataProcessor); + return stopPushProvideDataProcessor; + } + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java index 163a152da..3d798072a 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java @@ -16,12 +16,13 @@ */ package com.alipay.sofa.registry.server.session.bootstrap; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; +import java.util.concurrent.atomic.AtomicBoolean; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.SmartLifecycle; -import java.util.concurrent.atomic.AtomicBoolean; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; /** * SmartLifecycle for SessionServerBootstrap @@ -46,7 +47,7 @@ public boolean isAutoStartup() { @Override public void start() { try { - sessionServerBootstrap.doInitialized(); + sessionServerBootstrap.start(); LOGGER.info("Started SessionServer"); SessionServerInitializer.this.running.set(true); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheAccessException.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheAccessException.java new file mode 100644 index 000000000..7d43cbf65 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheAccessException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.cache; + +/** + * exception inner cache, e.g. remote invoke error + * @author kezhu.wukz + * @version $Id: CacheAccessException.java, v 0.1 2019-09-02 17:55 kezhu.wukz Exp $ + */ +public class CacheAccessException extends Exception { + public CacheAccessException() { + } + + public CacheAccessException(String message) { + super(message); + } + + public CacheAccessException(String message, Throwable cause) { + super(message, cause); + } + + public CacheAccessException(Throwable cause) { + super(cause); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheService.java index 6b95977ab..30c791f14 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheService.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/CacheService.java @@ -30,9 +30,9 @@ public interface CacheService { * @param key * @return */ - Value getValue(Key key); + Value getValue(Key key) throws CacheAccessException; - Map getValues(final Iterable keys); + Map getValues(final Iterable keys) throws CacheAccessException; /** * invalidate cache by keys diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/SessionCacheService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/SessionCacheService.java index ba5c1aab3..65f6bb842 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/SessionCacheService.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/cache/SessionCacheService.java @@ -16,17 +16,17 @@ */ package com.alipay.sofa.registry.server.session.cache; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; /** * @@ -84,23 +84,25 @@ private Value generatePayload(Key key) { } @Override - public Value getValue(final Key key) { + public Value getValue(final Key key) throws CacheAccessException { Value payload = null; try { payload = readWriteCacheMap.get(key); - } catch (Throwable t) { - LOGGER.error("Cannot get value for key :" + key, t); + } catch (Throwable e) { + String msg = "Cannot get value for key is:" + key; + throw new CacheAccessException(msg, e); } return payload; } @Override - public Map getValues(final Iterable keys) { + public Map getValues(final Iterable keys) throws CacheAccessException { Map valueMap = null; try { valueMap = readWriteCacheMap.getAll(keys); - } catch (ExecutionException e) { - LOGGER.error("Cannot get value for keys :" + keys, e); + } catch (Throwable e) { + String msg = "Cannot get value for keys are:" + keys; + throw new CacheAccessException(msg, e); } return valueMap; } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/connections/ConnectionsService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/connections/ConnectionsService.java new file mode 100644 index 000000000..c712c7d26 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/connections/ConnectionsService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.connections; + +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.resource.ConnectionsResource; +import com.alipay.sofa.registry.server.session.store.DataStore; +import com.alipay.sofa.registry.server.session.store.Interests; +import com.alipay.sofa.registry.server.session.store.Watchers; +import org.springframework.beans.factory.annotation.Autowired; + +import java.net.InetSocketAddress; +import java.util.*; +import java.util.stream.Collectors; + +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.CONNECT_ID_SPLIT; + +public class ConnectionsService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConnectionsResource.class); + + @Autowired + private Exchange boltExchange; + + @Autowired + private DataStore sessionDataStore; + + @Autowired + private Interests sessionInterests; + + @Autowired + private Watchers sessionWatchers; + + @Autowired + private SessionServerConfig sessionServerConfig; + + public List getConnections() { + Server server = boltExchange.getServer(sessionServerConfig.getServerPort()); + Set boltConnectIds = server.getChannels().stream().map( + channel -> channel.getRemoteAddress().getAddress().getHostAddress() + ":" + channel.getRemoteAddress().getPort() + ).collect(Collectors.toSet()); + Set connectIds = new HashSet<>(); + connectIds.addAll(sessionDataStore.getConnectPublishers().keySet().stream().map(connectId -> connectId.split(CONNECT_ID_SPLIT)[0]).collect(Collectors.toList())); + connectIds.addAll(sessionInterests.getConnectSubscribers().keySet().stream().map(connectId -> connectId.split(CONNECT_ID_SPLIT)[0]).collect(Collectors.toList())); + connectIds.addAll(sessionWatchers.getConnectWatchers().keySet().stream().map(connectId -> connectId.split(CONNECT_ID_SPLIT)[0]).collect(Collectors.toList())); + connectIds.retainAll(boltConnectIds); + return new ArrayList<>(connectIds); + } + + public void setMaxConnections(int connections) { + if (!sessionServerConfig.isEnableSessionLoadbalancePolicy()) { + throw new RuntimeException("drop connections is not allowed"); + } + List connectionIds = getConnections(); + + int needDropped = connectionIds.size() - connections; + if (needDropped <= 0) { + return; + } + Collections.shuffle(connectionIds); + connectionIds = connectionIds.subList(0, needDropped); + for (String ipAddress : connectionIds) { + Server server = boltExchange.getServer(sessionServerConfig.getServerPort()); + String[] parts = ipAddress.split(":"); + String host = parts[0]; + int port = Integer.parseInt(parts[1]); + InetSocketAddress address = new InetSocketAddress(host, port); + Channel connection = server.getChannel(address); + if (connection != null && connection.isConnected()) { + LOGGER.info("close connection {}", ipAddress); + server.close(connection); + } + } + } +} diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/converter/ReceivedDataConverter.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/converter/ReceivedDataConverter.java index 5280cc0a5..70fcc9f15 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/converter/ReceivedDataConverter.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/converter/ReceivedDataConverter.java @@ -16,6 +16,13 @@ */ package com.alipay.sofa.registry.server.session.converter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Predicate; + import com.alipay.sofa.registry.common.model.ServerDataBox; import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.dataserver.Datum; @@ -28,13 +35,7 @@ import com.alipay.sofa.registry.core.model.ScopeEnum; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.Predicate; +import com.alipay.sofa.registry.util.DatumVersionUtil; /** * The type Received data converter. @@ -173,7 +174,7 @@ public static ReceivedData getReceivedDataMulti(Map datums, Scope String regionLocal = subscriber.getCell(); receivedData.setLocalZone(regionLocal); - receivedData.setVersion(System.currentTimeMillis()); + receivedData.setVersion(DatumVersionUtil.nextId()); Map> swizzMap = new HashMap<>(); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/DataIdMatchStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/DataIdMatchStrategy.java new file mode 100644 index 000000000..4f06c85e1 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/DataIdMatchStrategy.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter; + +import java.util.function.Supplier; + +/** + * + * @author shangyu.wh + * @version 1.0: DataIdMatchStrategy.java, v 0.1 2019-06-19 22:12 shangyu.wh Exp $ + */ +public interface DataIdMatchStrategy { + + boolean match(String dataId, Supplier getOperatorType); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/IPMatchStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/IPMatchStrategy.java new file mode 100644 index 000000000..76697a5af --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/IPMatchStrategy.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter; + +import java.util.function.Supplier; + +/** + * + * @author shangyu.wh + * @version 1.0: IPMatchStrategy.java, v 0.1 2019-06-19 22:14 shangyu.wh Exp $ + */ +public interface IPMatchStrategy { + + boolean match(String IP, Supplier getOperatorType); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/ProcessFilter.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/ProcessFilter.java new file mode 100644 index 000000000..040db92c5 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/ProcessFilter.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter; + +/** + * + * @author shangyu.wh + * @version 1.0: ProcessFilter.java, v 0.1 2019-06-19 17:01 shangyu.wh Exp $ + */ +public interface ProcessFilter { + + boolean match(T input); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConfig.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConfig.java new file mode 100644 index 000000000..7e63e6bd2 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConfig.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import java.io.Serializable; +import java.util.List; + +/** + * + * @author shangyu.wh + * @version 1.0: BlacklistConfig.java, v 0.1 2019-06-19 17:23 shangyu.wh Exp $ + */ +public class BlacklistConfig implements Serializable { + + /** UID*/ + private static final long serialVersionUID = -7607561981062791932L; + + /** blacklist type */ + private String type; + + /** match type */ + @SuppressWarnings("rawtypes") + private List matchTypes; + + /** + * Getter method for property type. + * + * @return property value of type + */ + public String getType() { + return type; + } + + /** + * Setter method for property type. + * + * @param type value to be assigned to property type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Getter method for property matchTypes. + * + * @return property value of matchTypes + */ + @SuppressWarnings("rawtypes") + public List getMatchTypes() { + return matchTypes; + } + + /** + * Setter method for property matchTypes. + * + * @param matchTypes value to be assigned to property matchTypes + */ + @SuppressWarnings("rawtypes") + public void setMatchTypes(List matchTypes) { + this.matchTypes = matchTypes; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConstants.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConstants.java new file mode 100644 index 000000000..c5530f91d --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistConstants.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +/** + * + * @author shangyu.wh + * @version 1.0: BlacklistConstants.java, v 0.1 2019-06-19 17:56 shangyu.wh Exp $ + */ +public class BlacklistConstants { + + //*********************** Blacklist type ***********************// + /** forbidden pub */ + public static final String FORBIDDEN_PUB = "FORBIDDEN_PUB"; + + /** forbidden sub by prefix */ + public static final String FORBIDDEN_SUB_BY_PREFIX = "FORBIDDEN_SUB_BY_PREFIX"; + + //*********************** match type ***********************// + + /** match full ip */ + public static final String IP_FULL = "IP_FULL"; +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManager.java new file mode 100644 index 000000000..cf9a9733e --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManager.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * + * @author shangyu.wh + * @version 1.0: BlacklistManager.java, v 0.1 2019-06-19 18:27 shangyu.wh Exp $ + */ +public interface BlacklistManager { + /** + * load list first + */ + void load(); + + /** + * get list + * @return + */ + List getBlacklistConfigList(); + + /** + * set list + * @param blacklistConfigList + */ + void setBlacklistConfigList(List blacklistConfigList); + + /** + * convert blacklist map to blacklist config list + * @param config + * @return + * blacklistConfigMap key:blacklist type + * Map> key:MatchType + * set:match patterns + */ + Map>> convertBlacklistConfig(String config); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManagerImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManagerImpl.java new file mode 100644 index 000000000..f9ace0d5d --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistManagerImpl.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.FetchProvideDataRequest; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; +import com.alipay.sofa.registry.remoting.exchange.message.Request; +import com.alipay.sofa.registry.remoting.exchange.message.Response; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.node.RaftClientManager; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * + * @author shangyu.wh + * @version 1.0: BlacklistManagerImpl.java, v 0.1 2019-06-19 18:30 shangyu.wh Exp $ + */ +public class BlacklistManagerImpl implements BlacklistManager { + + private static final Logger LOGGER = LoggerFactory + .getLogger(BlacklistManagerImpl.class); + + private static final Logger EXCHANGE_LOGGER = LoggerFactory.getLogger("SESSION-EXCHANGE"); + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + protected NodeExchanger metaNodeExchanger; + + @Autowired + protected RaftClientManager raftClientManager; + + private List blacklistConfigList = new ArrayList(); + + @Override + public void load() { + fetchStopPushSwitch(); + } + + @Override + public List getBlacklistConfigList() { + return blacklistConfigList; + } + + @Override + public void setBlacklistConfigList(List blacklistConfigList) { + this.blacklistConfigList = blacklistConfigList; + } + + private void fetchStopPushSwitch() { + + Object ret = sendMetaRequest(); + if (ret instanceof ProvideData) { + ProvideData provideData = (ProvideData) ret; + if (provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER.info("Fetch session blacklist no data existed,current config not change!"); + return; + } + String data = (String) provideData.getProvideData().getObject(); + if (data != null) { + convertBlacklistConfig(data); + EXCHANGE_LOGGER.info("Fetch session blacklist data switch {} success!", data); + } else { + LOGGER.info("Fetch session blacklist data null,current config not change!"); + } + } else { + LOGGER.info("Fetch session blacklist data null,config not change!"); + } + } + + private Object sendMetaRequest() { + try { + Request request = new Request() { + @Override + public FetchProvideDataRequest getRequestBody() { + return new FetchProvideDataRequest(ValueConstants.BLACK_LIST_DATA_ID); + } + + @Override + public URL getRequestUrl() { + return new URL(raftClientManager.getLeader().getIp(), + sessionServerConfig.getMetaServerPort()); + } + }; + + Response response = metaNodeExchanger.request(request); + + if (response != null && response.getResult() != null) { + EXCHANGE_LOGGER.info("Update blacklist info success!"); + return response.getResult(); + } else { + LOGGER.error("Get blacklist info error!No response receive!"); + throw new RuntimeException("Get blacklist info error!No response receive!"); + } + } catch (Exception e) { + LOGGER.error("Get blacklist info error!", e); + throw new RuntimeException("Get blacklist info error! ", e); + } + } + + /** + * @see BlacklistManager#convertBlacklistConfig(String) + */ + public Map>> convertBlacklistConfig(String config) { + + TypeReference>>> typeReference = new TypeReference>>>() { + }; + + ObjectMapper mapper = new ObjectMapper(); + + Map>> blacklistConfigMap; + try { + blacklistConfigMap = mapper.readValue(config, typeReference); + } catch (IOException e) { + LOGGER.error("Parser config json error!", e); + return null; + } + if (null == blacklistConfigMap) { + LOGGER.info("[cmd] setBlacklistConfig fail, params is null"); + return null; + } + try { + List blacklistConfigs = new ArrayList(); + for (Entry>> configEntry : blacklistConfigMap + .entrySet()) { + BlacklistConfig blacklistConfig = new BlacklistConfig(); + blacklistConfig.setType(configEntry.getKey()); + + List matchTypeList = new ArrayList(); + + Map> matchTypeMap = configEntry.getValue(); + for (Entry> typeEntry : matchTypeMap.entrySet()) { + String type = typeEntry.getKey(); + + MatchType ipFullMatchType = new MatchType(); + ipFullMatchType.setType(type); + ipFullMatchType.setPatternSet(typeEntry.getValue()); + matchTypeList.add(ipFullMatchType); + } + blacklistConfig.setMatchTypes(matchTypeList); + blacklistConfigs.add(blacklistConfig); + } + + setBlacklistConfigList(blacklistConfigs); + return blacklistConfigMap; + } catch (Exception e) { + LOGGER.error("[cmd] setBlacklistConfig error", e); + return null; + } + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistMatchProcessFilter.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistMatchProcessFilter.java new file mode 100644 index 000000000..5f1845915 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/BlacklistMatchProcessFilter.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import com.alipay.sofa.registry.common.model.store.BaseInfo; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.filter.DataIdMatchStrategy; +import com.alipay.sofa.registry.server.session.filter.IPMatchStrategy; +import com.alipay.sofa.registry.server.session.filter.ProcessFilter; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; + +/** + * + * @author shangyu.wh + * @version 1.0: BlacklistMatchProcessFilter.java, v 0.1 2019-06-19 22:01 shangyu.wh Exp $ + */ +public class BlacklistMatchProcessFilter implements ProcessFilter { + + @Autowired + private DataIdMatchStrategy dataIdMatchStrategy; + + @Autowired + private IPMatchStrategy ipMatchStrategy; + + @Autowired + private BlacklistManager blacklistManager; + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Override + public boolean match(BaseInfo storeData) { + + final List configList = blacklistManager.getBlacklistConfigList(); + + // empty list proceed + if (null == configList || configList.size() == 0) { + return false; + } + + URL url = storeData.getSourceAddress(); + + if (url != null) { + + switch (storeData.getDataType()) { + case PUBLISHER: + if(dataIdMatchStrategy.match(storeData.getDataId(),()-> sessionServerConfig.getBlacklistPubDataIdRegex())){ + return ipMatchStrategy.match(url.getIpAddress(),()-> BlacklistConstants.FORBIDDEN_PUB); + } + case SUBSCRIBER: + if(dataIdMatchStrategy.match(storeData.getDataId(),()-> sessionServerConfig.getBlacklistSubDataIdRegex())){ + return ipMatchStrategy.match(url.getIpAddress(),()-> BlacklistConstants.FORBIDDEN_SUB_BY_PREFIX); + } + default: + return false; + } + + } + return false; + } + + /** + * Getter method for property dataIdMatchStrategy. + * + * @return property value of dataIdMatchStrategy + */ + public DataIdMatchStrategy getDataIdMatchStrategy() { + return dataIdMatchStrategy; + } + + /** + * Setter method for property dataIdMatchStrategy. + * + * @param dataIdMatchStrategy value to be assigned to property dataIdMatchStrategy + */ + public void setDataIdMatchStrategy(DataIdMatchStrategy dataIdMatchStrategy) { + this.dataIdMatchStrategy = dataIdMatchStrategy; + } + + /** + * Getter method for property ipMatchStrategy. + * + * @return property value of ipMatchStrategy + */ + public IPMatchStrategy getIpMatchStrategy() { + return ipMatchStrategy; + } + + /** + * Setter method for property ipMatchStrategy. + * + * @param ipMatchStrategy value to be assigned to property ipMatchStrategy + */ + public void setIpMatchStrategy(IPMatchStrategy ipMatchStrategy) { + this.ipMatchStrategy = ipMatchStrategy; + } + + /** + * Getter method for property blacklistManager. + * + * @return property value of blacklistManager + */ + public BlacklistManager getBlacklistManager() { + return blacklistManager; + } + + /** + * Setter method for property blacklistManager. + * + * @param blacklistManager value to be assigned to property blacklistManager + */ + public void setBlacklistManager(BlacklistManager blacklistManager) { + this.blacklistManager = blacklistManager; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultDataIdMatchStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultDataIdMatchStrategy.java new file mode 100644 index 000000000..23abfaecd --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultDataIdMatchStrategy.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import com.alipay.sofa.registry.server.session.filter.DataIdMatchStrategy; +import org.springframework.util.StringUtils; + +import java.util.function.Supplier; +import java.util.regex.Pattern; + +/** + * + * @author shangyu.wh + * @version 1.0: DefaultDataIdMatchStrategy.java, v 0.1 2019-06-19 22:15 shangyu.wh Exp $ + */ +public class DefaultDataIdMatchStrategy implements DataIdMatchStrategy { + + @Override + public boolean match(String dataId, Supplier getOperatorType) { + if (StringUtils.hasText(dataId) && StringUtils.hasText(getOperatorType.get())) { + return Pattern.matches(getOperatorType.get(), dataId); + } + return true; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultIPMatchStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultIPMatchStrategy.java new file mode 100644 index 000000000..f4289f374 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/DefaultIPMatchStrategy.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import com.alipay.sofa.registry.server.session.filter.IPMatchStrategy; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +/** + * + * @author shangyu.wh + * @version 1.0: DefaultIPMatchStrategy.java, v 0.1 2019-06-19 22:16 shangyu.wh Exp $ + */ +public class DefaultIPMatchStrategy implements IPMatchStrategy { + + @Autowired + private BlacklistManager blacklistManager; + + @Override + public boolean match(String IP, Supplier getOperatorType) { + return match(getOperatorType.get(), IP); + } + + private boolean match(String type, String matchPattern) { + + List configList = blacklistManager.getBlacklistConfigList(); + for (BlacklistConfig blacklistConfig : configList) { + + // 如黑名单类型不匹配则跳过 + if (!StringUtils.equals(type, blacklistConfig.getType())) { + continue; + } + + List matchTypeList = blacklistConfig.getMatchTypes(); + + // 匹配规则为空跳过 + if (null == matchTypeList || matchTypeList.size() == 0) { + continue; + } + + for (MatchType matchType : matchTypeList) { + if (null == matchType) { + continue; + } + + if (BlacklistConstants.IP_FULL.equals(matchType.getType())) { + // IP 全匹配时判断当前发布者IP是否在IP列表中，如命中则拒绝发布 + @SuppressWarnings("unchecked") + Set patterns = matchType.getPatternSet(); + + if (null == patterns || patterns.size() == 0) { + continue; + } + + if (patterns.contains(matchPattern)) { + return true; + } + } + } + } + return false; + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/MatchType.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/MatchType.java new file mode 100644 index 000000000..6a1318770 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/filter/blacklist/MatchType.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.filter.blacklist; + +import java.io.Serializable; +import java.util.Set; + +/** + * + * @author shangyu.wh + * @version 1.0: MatchType.java, v 0.1 2019-06-19 17:25 shangyu.wh Exp $ + */ +public class MatchType implements Serializable { + /** UID */ + private static final long serialVersionUID = 4015181538538056685L; + + /** match type */ + private String type; + + /** match patterns */ + private Set patternSet; + + /** + * Getter method for property type. + * + * @return property value of type + */ + public String getType() { + return type; + } + + /** + * Setter method for property type. + * + * @param type value to be assigned to property type + */ + public void setType(String type) { + this.type = type; + } + + /** + * Getter method for property patternSet. + * + * @return property value of patternSet + */ + public Set getPatternSet() { + return patternSet; + } + + /** + * Setter method for property patternSet. + * + * @param patternSet value to be assigned to property patternSet + */ + public void setPatternSet(Set patternSet) { + this.patternSet = patternSet; + } + + /** + * @see Object#toString() + */ + @Override + public String toString() { + return "MatchType{" + "type='" + type + '\'' + ", patternSet=" + patternSet + '}'; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitService.java new file mode 100644 index 000000000..7ac67b0f2 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitService.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.limit; + +/** + * + * @author shangyu.wh + * @version 1.0: AccessLimitService.java, v 0.1 2019-08-26 20:35 shangyu.wh Exp $ + */ +public interface AccessLimitService { + boolean tryAcquire(); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitServiceImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitServiceImpl.java new file mode 100644 index 000000000..fef1c6a8f --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/limit/AccessLimitServiceImpl.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.limit; + +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.google.common.util.concurrent.RateLimiter; + +/** + * + * @author shangyu.wh + * @version 1.0: AccessLimitServiceImpl.java, v 0.1 2019-08-26 20:37 shangyu.wh Exp $ + */ +public class AccessLimitServiceImpl implements AccessLimitService { + + private SessionServerConfig sessionServerConfig; + + private RateLimiter rateLimiter; + + public AccessLimitServiceImpl(SessionServerConfig sessionServerConfig) { + this.sessionServerConfig = sessionServerConfig; + rateLimiter = RateLimiter.create(sessionServerConfig.getAccessLimitRate()); + } + + @Override + public boolean tryAcquire() { + return rateLimiter.tryAcquire(); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/CancelDataTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/CancelDataTaskListener.java index 0bae12a2e..0197fd8b5 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/CancelDataTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/CancelDataTaskListener.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.session.listener; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.DataNodeService; import com.alipay.sofa.registry.server.session.scheduler.task.CancelDataTask; @@ -29,7 +33,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -67,18 +70,16 @@ public class CancelDataTaskListener implements TaskListener { @Autowired private TaskProcessor dataNodeSingleTaskProcessor; - public TaskDispatcher getSingleTaskDispatcher() { - if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( - TaskDispatchers.getDispatcherName(TaskType.CANCEL_DATA_TASK.getName()), 10000, 80, - 1000, 100, dataNodeSingleTaskProcessor); - } - return singleTaskDispatcher; + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.CANCEL_DATA_TASK.getName()), 10000, 80, + 1000, 100, dataNodeSingleTaskProcessor); } @Override - public boolean support(TaskEvent event) { - return TaskType.CANCEL_DATA_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.CANCEL_DATA_TASK; } @Override @@ -88,7 +89,8 @@ public void handleEvent(TaskEvent event) { sessionWatchers, dataNodeService, sessionServerConfig); cancelDataTask.setTaskEvent(event); - getSingleTaskDispatcher().dispatch(cancelDataTask.getTaskId(), cancelDataTask, + singleTaskDispatcher.dispatch(cancelDataTask.getTaskId(), cancelDataTask, cancelDataTask.getExpiryTime()); } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchCloudTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchCloudTaskListener.java index e558c98f5..65674b062 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchCloudTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchCloudTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; @@ -29,7 +31,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -39,26 +40,26 @@ public class DataChangeFetchCloudTaskListener implements TaskListener { @Autowired - private Interests sessionInterests; + private Interests sessionInterests; @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; /** * trigger task com.alipay.sofa.registry.server.meta.listener process */ @Autowired - private TaskListenerManager taskListenerManager; + private TaskListenerManager taskListenerManager; @Autowired - private ExecutorManager executorManager; + private ExecutorManager executorManager; @Autowired - private CacheService sessionCacheService; + private CacheService sessionCacheService; - private TaskDispatcher singleTaskDispatcher; + private volatile TaskDispatcher singleTaskDispatcher; - private TaskProcessor dataNodeSingleTaskProcessor; + private TaskProcessor dataNodeSingleTaskProcessor; public DataChangeFetchCloudTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; @@ -66,18 +67,22 @@ public DataChangeFetchCloudTaskListener(TaskProcessor dataNodeSingleTaskProcesso public TaskDispatcher getSingleTaskDispatcher() { if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( - TaskDispatchers.getDispatcherName(TaskType.DATA_CHANGE_FETCH_CLOUD_TASK.getName()), - sessionServerConfig.getDataChangeFetchTaskMaxBufferSize(), - sessionServerConfig.getDataChangeFetchTaskWorkerSize(), 1000, 100, - dataNodeSingleTaskProcessor); + synchronized (this) { + if (singleTaskDispatcher == null) { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.DATA_CHANGE_FETCH_CLOUD_TASK + .getName()), sessionServerConfig.getDataChangeFetchTaskMaxBufferSize(), + sessionServerConfig.getDataChangeFetchTaskWorkerSize(), 1000, 100, + dataNodeSingleTaskProcessor); + } + } } return singleTaskDispatcher; } @Override - public boolean support(TaskEvent event) { - return TaskType.DATA_CHANGE_FETCH_CLOUD_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.DATA_CHANGE_FETCH_CLOUD_TASK; } @Override @@ -88,4 +93,5 @@ public void handleEvent(TaskEvent event) { getSingleTaskDispatcher().dispatch(dataChangeFetchTask.getTaskId(), dataChangeFetchTask, dataChangeFetchTask.getExpiryTime()); } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchTaskListener.java index 434e6a61c..a2517509a 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataChangeFetchTaskListener.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.session.listener; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; @@ -29,7 +33,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -38,27 +41,30 @@ */ public class DataChangeFetchTaskListener implements TaskListener { + private final static Logger LOGGER = LoggerFactory + .getLogger(DataChangeFetchTaskListener.class); + @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; @Autowired - private Interests sessionInterests; + private Interests sessionInterests; @Autowired - private ExecutorManager executorManager; + private ExecutorManager executorManager; @Autowired - private CacheService sessionCacheService; + private CacheService sessionCacheService; /** * trigger task com.alipay.sofa.registry.server.meta.listener process */ @Autowired - private TaskListenerManager taskListenerManager; + private TaskListenerManager taskListenerManager; - private TaskDispatcher singleTaskDispatcher; + private volatile TaskDispatcher singleTaskDispatcher; - private TaskProcessor dataNodeSingleTaskProcessor; + private TaskProcessor dataNodeSingleTaskProcessor; public DataChangeFetchTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; @@ -66,18 +72,23 @@ public DataChangeFetchTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { public TaskDispatcher getSingleTaskDispatcher() { if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( - TaskDispatchers.getDispatcherName(TaskType.DATA_CHANGE_FETCH_TASK.getName()), - sessionServerConfig.getDataChangeFetchTaskMaxBufferSize(), - sessionServerConfig.getDataChangeFetchTaskWorkerSize(), 1000, 100, - dataNodeSingleTaskProcessor); + synchronized (this) { + if (singleTaskDispatcher == null) { + singleTaskDispatcher = TaskDispatchers + .createSingleTaskDispatcher(TaskDispatchers + .getDispatcherName(TaskType.DATA_CHANGE_FETCH_TASK.getName()), + sessionServerConfig.getDataChangeFetchTaskMaxBufferSize(), + sessionServerConfig.getDataChangeFetchTaskWorkerSize(), 1000, 100, + dataNodeSingleTaskProcessor); + } + } } return singleTaskDispatcher; } @Override - public boolean support(TaskEvent event) { - return TaskType.DATA_CHANGE_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.DATA_CHANGE_FETCH_TASK; } @Override @@ -85,8 +96,10 @@ public void handleEvent(TaskEvent event) { SessionTask dataChangeFetchTask = new DataChangeFetchTask(sessionServerConfig, taskListenerManager, executorManager, sessionInterests, sessionCacheService); dataChangeFetchTask.setTaskEvent(event); + getSingleTaskDispatcher().dispatch(dataChangeFetchTask.getTaskId(), dataChangeFetchTask, dataChangeFetchTask.getExpiryTime()); + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataPushTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataPushTaskListener.java index c1afe3314..f27db140e 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataPushTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DataPushTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.scheduler.task.DataPushTask; @@ -28,7 +30,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -38,23 +39,23 @@ public class DataPushTaskListener implements TaskListener { @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; @Autowired - private Interests sessionInterests; + private Interests sessionInterests; /** * trigger task com.alipay.sofa.registry.server.meta.listener process */ @Autowired - private TaskListenerManager taskListenerManager; + private TaskListenerManager taskListenerManager; @Autowired - private ExecutorManager executorManager; + private ExecutorManager executorManager; - private TaskDispatcher singleTaskDispatcher; + private volatile TaskDispatcher singleTaskDispatcher; - private TaskProcessor dataNodeSingleTaskProcessor; + private TaskProcessor dataNodeSingleTaskProcessor; public DataPushTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { @@ -63,15 +64,19 @@ public DataPushTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { public TaskDispatcher getSingleTaskDispatcher() { if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( - TaskType.DATA_PUSH_TASK.getName(), dataNodeSingleTaskProcessor); + synchronized (this) { + if (singleTaskDispatcher == null) { + singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( + TaskType.DATA_PUSH_TASK.getName(), dataNodeSingleTaskProcessor); + } + } } return singleTaskDispatcher; } @Override - public boolean support(TaskEvent event) { - return TaskType.DATA_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.DATA_PUSH_TASK; } @Override diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DatumSnapshotTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DatumSnapshotTaskListener.java new file mode 100644 index 000000000..b86795cba --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/DatumSnapshotTaskListener.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.listener; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.scheduler.task.DatumSnapshotTask; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.task.batcher.TaskDispatcher; +import com.alipay.sofa.registry.task.batcher.TaskDispatchers; +import com.alipay.sofa.registry.task.batcher.TaskProcessor; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListener; + +/** + * + * @author kezhu.wukz + * @version $Id: RenewDatumTaskListener.java, v 0.1 2019-06-14 12:02 kezhu.wukz Exp $ + */ +public class DatumSnapshotTaskListener implements TaskListener { + + @Autowired + private DataNodeService dataNodeService; + + private TaskDispatcher singleTaskDispatcher; + + @Autowired + private TaskProcessor dataNodeSingleTaskProcessor; + + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.DATUM_SNAPSHOT_TASK.getName()), 100000, 32, + 1000, 1000, dataNodeSingleTaskProcessor); + } + + @Override + public TaskType support() { + return TaskType.DATUM_SNAPSHOT_TASK; + } + + @Override + public void handleEvent(TaskEvent event) { + SessionTask datumSnapshotTask = new DatumSnapshotTask(dataNodeService); + + datumSnapshotTask.setTaskEvent(event); + + singleTaskDispatcher.dispatch(datumSnapshotTask.getTaskId(), datumSnapshotTask, + datumSnapshotTask.getExpiryTime()); + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ProvideDataChangeFetchTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ProvideDataChangeFetchTaskListener.java index fe4b73208..234030672 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ProvideDataChangeFetchTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ProvideDataChangeFetchTaskListener.java @@ -16,13 +16,16 @@ */ package com.alipay.sofa.registry.server.session.listener; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.MetaNodeService; -import com.alipay.sofa.registry.server.session.registry.Registry; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; import com.alipay.sofa.registry.server.session.scheduler.task.ProvideDataChangeFetchTask; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; -import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.store.Watchers; import com.alipay.sofa.registry.task.batcher.TaskDispatcher; import com.alipay.sofa.registry.task.batcher.TaskDispatchers; @@ -31,7 +34,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -58,14 +60,11 @@ public class ProvideDataChangeFetchTaskListener implements TaskListener { @Autowired private Exchange boltExchange; - @Autowired - private Interests sessionInterests; - @Autowired private Watchers sessionWatchers; @Autowired - private Registry sessionRegistry; + private ProvideDataProcessor provideDataProcessorManager; private TaskDispatcher singleTaskDispatcher; @@ -75,17 +74,15 @@ public ProvideDataChangeFetchTaskListener(TaskProcessor dataNodeSingleTaskProces this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; } - public TaskDispatcher getSingleTaskDispatcher() { - if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( - TaskType.PROVIDE_DATA_CHANGE_FETCH_TASK.getName(), dataNodeSingleTaskProcessor); - } - return singleTaskDispatcher; + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( + TaskType.PROVIDE_DATA_CHANGE_FETCH_TASK.getName(), dataNodeSingleTaskProcessor); } @Override - public boolean support(TaskEvent event) { - return TaskType.PROVIDE_DATA_CHANGE_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.PROVIDE_DATA_CHANGE_FETCH_TASK; } @Override @@ -93,11 +90,11 @@ public void handleEvent(TaskEvent event) { SessionTask provideDataChangeFetchTask = new ProvideDataChangeFetchTask( sessionServerConfig, taskListenerManager, metaNodeService, sessionWatchers, - boltExchange, sessionInterests, sessionRegistry); + boltExchange, provideDataProcessorManager); provideDataChangeFetchTask.setTaskEvent(event); - getSingleTaskDispatcher().dispatch(provideDataChangeFetchTask.getTaskId(), + singleTaskDispatcher.dispatch(provideDataChangeFetchTask.getTaskId(), provideDataChangeFetchTask, provideDataChangeFetchTask.getExpiryTime()); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/PublishDataTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/PublishDataTaskListener.java new file mode 100644 index 000000000..5b5d8be7d --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/PublishDataTaskListener.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.listener; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import com.alipay.sofa.registry.server.session.scheduler.task.PublishDataTask; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.task.batcher.TaskProcessor; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListener; + +/** + * + * @author kezhu.wukz + * @version $Id: PublishDataTaskListener.java, v 0.1 2019-06-17 12:02 kezhu.wukz Exp $ + */ +public class PublishDataTaskListener implements TaskListener { + + @Autowired + private DataNodeService dataNodeService; + + @Autowired + private TaskProcessor dataNodeSingleTaskProcessor; + + @Autowired + private ExecutorManager executorManager; + + @Override + public TaskType support() { + return TaskType.PUBLISH_DATA_TASK; + } + + @Override + public void handleEvent(TaskEvent event) { + + SessionTask publishDataTask = new PublishDataTask(dataNodeService); + + publishDataTask.setTaskEvent(event); + + executorManager.getPublishDataExecutor().execute(()-> dataNodeSingleTaskProcessor.process(publishDataTask)); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedConfigDataPushTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedConfigDataPushTaskListener.java index 17cc54b5e..5103ae201 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedConfigDataPushTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedConfigDataPushTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.ClientNodeService; import com.alipay.sofa.registry.server.session.scheduler.task.ReceivedConfigDataPushTask; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -37,17 +38,17 @@ public class ReceivedConfigDataPushTaskListener implements TaskListener { @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; @Autowired - private ClientNodeService clientNodeService; + private ClientNodeService clientNodeService; @Autowired - private ReceivedConfigDataPushTaskStrategy receivedConfigDataPushTaskStrategy; + private ReceivedConfigDataPushTaskStrategy receivedConfigDataPushTaskStrategy; - private TaskDispatcher singleTaskDispatcher; + private volatile TaskDispatcher singleTaskDispatcher; - private TaskProcessor clientNodeSingleTaskProcessor; + private TaskProcessor clientNodeSingleTaskProcessor; public ReceivedConfigDataPushTaskListener(TaskProcessor clientNodeSingleTaskProcessor) { @@ -56,15 +57,20 @@ public ReceivedConfigDataPushTaskListener(TaskProcessor clientNodeSingleTaskProc public TaskDispatcher getSingleTaskDispatcher() { if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( - TaskType.RECEIVED_DATA_CONFIG_PUSH_TASK.getName(), clientNodeSingleTaskProcessor); + synchronized (this) { + if (singleTaskDispatcher == null) { + singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( + TaskType.RECEIVED_DATA_CONFIG_PUSH_TASK.getName(), + clientNodeSingleTaskProcessor); + } + } } return singleTaskDispatcher; } @Override - public boolean support(TaskEvent event) { - return TaskType.RECEIVED_DATA_CONFIG_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.RECEIVED_DATA_CONFIG_PUSH_TASK; } @Override diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedDataMultiPushTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedDataMultiPushTaskListener.java index e4c8b06b3..3d036f647 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedDataMultiPushTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/ReceivedDataMultiPushTaskListener.java @@ -16,16 +16,23 @@ */ package com.alipay.sofa.registry.server.session.listener; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.ClientNodeService; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import com.alipay.sofa.registry.server.session.scheduler.task.PushTaskClosure; import com.alipay.sofa.registry.server.session.scheduler.task.ReceivedDataMultiPushTask; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.strategy.ReceivedDataMultiPushTaskStrategy; import com.alipay.sofa.registry.server.session.strategy.TaskMergeProcessorStrategy; +import com.alipay.sofa.registry.task.TaskClosure; import com.alipay.sofa.registry.task.batcher.TaskProcessor; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; @@ -33,9 +40,6 @@ import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.concurrent.TimeUnit; /** * @@ -62,6 +66,9 @@ public class ReceivedDataMultiPushTaskListener implements TaskListener, PushTask @Autowired private ReceivedDataMultiPushTaskStrategy receivedDataMultiPushTaskStrategy; + @Autowired + private Interests sessionInterests; + private TaskMergeProcessorStrategy receiveDataTaskMergeProcessorStrategy; private TaskProcessor clientNodeSingleTaskProcessor; @@ -78,7 +85,9 @@ public ReceivedDataMultiPushTaskListener(TaskProcessor clientNodeSingleTaskProce asyncHashedWheelTimer = new AsyncHashedWheelTimer(threadFactoryBuilder.setNameFormat( "Registry-ReceivedDataPushTask-WheelTimer").build(), sessionServerConfig.getUserDataPushRetryWheelTicksDuration(), TimeUnit.MILLISECONDS, - sessionServerConfig.getUserDataPushRetryWheelTicksSize(), threadFactoryBuilder + sessionServerConfig.getUserDataPushRetryWheelTicksSize(), + sessionServerConfig.getUserDataPushRetryExecutorThreadSize(), + sessionServerConfig.getUserDataPushRetryExecutorQueueSize(), threadFactoryBuilder .setNameFormat("Registry-ReceivedDataPushTask-WheelExecutor-%d").build(), new TaskFailedCallback() { @Override @@ -97,12 +106,17 @@ public void executionFailed(Throwable e) { } @Override - public boolean support(TaskEvent event) { - return TaskType.RECEIVED_DATA_MULTI_PUSH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.RECEIVED_DATA_MULTI_PUSH_TASK; } @Override public void handleEvent(TaskEvent event) { + TaskClosure taskClosure = event.getTaskClosure(); + + if (taskClosure != null && taskClosure instanceof PushTaskClosure) { + ((PushTaskClosure) taskClosure).addTask(event); + } receiveDataTaskMergeProcessorStrategy.handleEvent(event); } @@ -110,7 +124,7 @@ public void handleEvent(TaskEvent event) { public void executePushAsync(TaskEvent event) { SessionTask receivedDataMultiPushTask = new ReceivedDataMultiPushTask(sessionServerConfig, clientNodeService, - executorManager, boltExchange, receivedDataMultiPushTaskStrategy,asyncHashedWheelTimer); + executorManager, boltExchange, receivedDataMultiPushTaskStrategy,asyncHashedWheelTimer,sessionInterests); receivedDataMultiPushTask.setTaskEvent(event); executorManager.getPushTaskExecutor() diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/RenewDatumTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/RenewDatumTaskListener.java new file mode 100644 index 000000000..d1624659d --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/RenewDatumTaskListener.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.listener; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.registry.SessionRegistry; +import com.alipay.sofa.registry.server.session.scheduler.task.RenewDatumTask; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.task.batcher.TaskDispatcher; +import com.alipay.sofa.registry.task.batcher.TaskDispatchers; +import com.alipay.sofa.registry.task.batcher.TaskProcessor; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListener; + +/** + * + * @author kezhu.wukz + * @version $Id: RenewDatumTaskListener.java, v 0.1 2019-06-17 12:02 kezhu.wukz Exp $ + */ +public class RenewDatumTaskListener implements TaskListener { + + @Autowired + private DataNodeService dataNodeService; + + @Autowired + private SessionServerConfig sessionServerConfig; + + private TaskDispatcher singleTaskDispatcher; + + @Autowired + private TaskProcessor dataNodeSingleTaskProcessor; + + @Autowired + private SessionRegistry sessionRegistry; + + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.RENEW_DATUM_TASK.getName()), 100000, 32, + 1000, 1000, dataNodeSingleTaskProcessor); + } + + @Override + public TaskType support() { + return TaskType.RENEW_DATUM_TASK; + } + + @Override + public void handleEvent(TaskEvent event) { + SessionTask renewDatumTask = new RenewDatumTask(sessionServerConfig, dataNodeService, + sessionRegistry); + + renewDatumTask.setTaskEvent(event); + + singleTaskDispatcher.dispatch(renewDatumTask.getTaskId(), renewDatumTask, + renewDatumTask.getExpiryTime()); + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SessionRegisterDataTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SessionRegisterDataTaskListener.java index 38563b430..2cec75755 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SessionRegisterDataTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SessionRegisterDataTaskListener.java @@ -16,9 +16,12 @@ */ package com.alipay.sofa.registry.server.session.listener; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import com.alipay.sofa.registry.server.session.node.service.DataNodeService; import com.alipay.sofa.registry.server.session.scheduler.task.SessionRegisterDataTask; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; import com.alipay.sofa.registry.task.batcher.TaskDispatcher; @@ -27,7 +30,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -36,12 +38,6 @@ */ public class SessionRegisterDataTaskListener implements TaskListener { - /** - * DataNode service - */ - @Autowired - private DataNodeService dataNodeService; - @Autowired private Exchange boltExchange; @@ -57,26 +53,24 @@ public SessionRegisterDataTaskListener(TaskProcessor dataNodeSingleTaskProcessor this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; } - public TaskDispatcher getSingleTaskDispatcher() { - if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( - TaskDispatchers.getDispatcherName(TaskType.SESSION_REGISTER_DATA_TASK.getName()), - 60, 5, 1000, 100, dataNodeSingleTaskProcessor); - } - return singleTaskDispatcher; + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.SESSION_REGISTER_DATA_TASK.getName()), 60, + 5, 1000, 100, dataNodeSingleTaskProcessor); } @Override - public boolean support(TaskEvent event) { - return TaskType.SESSION_REGISTER_DATA_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.SESSION_REGISTER_DATA_TASK; } @Override public void handleEvent(TaskEvent event) { SessionTask sessionRegisterDataTask = new SessionRegisterDataTask(boltExchange, - dataNodeService, sessionServerConfig); + sessionServerConfig); sessionRegisterDataTask.setTaskEvent(event); - getSingleTaskDispatcher().dispatch(sessionRegisterDataTask.getTaskId(), - sessionRegisterDataTask, sessionRegisterDataTask.getExpiryTime()); + singleTaskDispatcher.dispatch(sessionRegisterDataTask.getTaskId(), sessionRegisterDataTask, + sessionRegisterDataTask.getExpiryTime()); } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberMultiFetchTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberMultiFetchTaskListener.java index 773e5e367..827f20fab 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberMultiFetchTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberMultiFetchTaskListener.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.session.listener; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; @@ -28,7 +32,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -61,18 +64,16 @@ public SubscriberMultiFetchTaskListener(TaskProcessor dataNodeSingleTaskProcesso this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; } - public TaskDispatcher getSingleTaskDispatcher() { - if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( - TaskDispatchers.getDispatcherName(TaskType.SUBSCRIBER_MULTI_FETCH_TASK.getName()), - 100000, 80, 0, 0, dataNodeSingleTaskProcessor); - } - return singleTaskDispatcher; + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.SUBSCRIBER_MULTI_FETCH_TASK.getName()), + 100000, 80, 0, 0, dataNodeSingleTaskProcessor); } @Override - public boolean support(TaskEvent event) { - return TaskType.SUBSCRIBER_MULTI_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.SUBSCRIBER_MULTI_FETCH_TASK; } @Override @@ -83,7 +84,7 @@ public void handleEvent(TaskEvent event) { subscriberMultiFetchTask.setTaskEvent(event); - getSingleTaskDispatcher().dispatch(subscriberMultiFetchTask.getTaskId(), + singleTaskDispatcher.dispatch(subscriberMultiFetchTask.getTaskId(), subscriberMultiFetchTask, subscriberMultiFetchTask.getExpiryTime()); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberPushEmptyTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberPushEmptyTaskListener.java new file mode 100644 index 000000000..d3b36e119 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberPushEmptyTaskListener.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.listener; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.server.session.scheduler.task.SubscriberPushEmptyTask; +import com.alipay.sofa.registry.task.batcher.TaskDispatcher; +import com.alipay.sofa.registry.task.batcher.TaskDispatchers; +import com.alipay.sofa.registry.task.batcher.TaskProcessor; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListener; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; + +/** + * + * @author shangyu.wh + * @version $Id: SubscriberRegisterFetchTaskListener.java, v 0.1 2017-12-07 19:53 shangyu.wh Exp $ + */ +public class SubscriberPushEmptyTaskListener implements TaskListener { + + @Autowired + private SessionServerConfig sessionServerConfig; + + /** + * trigger task com.alipay.sofa.registry.server.meta.listener process + */ + @Autowired + private TaskListenerManager taskListenerManager; + + private TaskDispatcher singleTaskDispatcher; + + @Autowired + private TaskProcessor dataNodeSingleTaskProcessor; + + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( + TaskType.SUBSCRIBER_PUSH_EMPTY_TASK.getName(), dataNodeSingleTaskProcessor); + } + + @Override + public TaskType support() { + return TaskType.SUBSCRIBER_PUSH_EMPTY_TASK; + } + + @Override + public void handleEvent(TaskEvent event) { + + SessionTask subscriberPushEmptyTask = new SubscriberPushEmptyTask(sessionServerConfig, + taskListenerManager); + + subscriberPushEmptyTask.setTaskEvent(event); + + singleTaskDispatcher.dispatch(subscriberPushEmptyTask.getTaskId(), subscriberPushEmptyTask, + subscriberPushEmptyTask.getExpiryTime()); + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberRegisterFetchTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberRegisterFetchTaskListener.java index 68b66cf63..6feb653a2 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberRegisterFetchTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/SubscriberRegisterFetchTaskListener.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.session.listener; +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.node.service.DataNodeService; @@ -29,7 +33,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -67,18 +70,16 @@ public SubscriberRegisterFetchTaskListener(TaskProcessor dataNodeSingleTaskProce this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; } - public TaskDispatcher getSingleTaskDispatcher() { - if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher(TaskDispatchers - .getDispatcherName(TaskType.SUBSCRIBER_REGISTER_FETCH_TASK.getName()), 200000, 80, - 1000, 100, dataNodeSingleTaskProcessor); - } - return singleTaskDispatcher; + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.SUBSCRIBER_REGISTER_FETCH_TASK.getName()), + 200000, 80, 1000, 100, dataNodeSingleTaskProcessor); } @Override - public boolean support(TaskEvent event) { - return TaskType.SUBSCRIBER_REGISTER_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.SUBSCRIBER_REGISTER_FETCH_TASK; } @Override @@ -90,7 +91,7 @@ public void handleEvent(TaskEvent event) { subscriberRegisterFetchTask.setTaskEvent(event); - getSingleTaskDispatcher().dispatch(subscriberRegisterFetchTask.getTaskId(), + singleTaskDispatcher.dispatch(subscriberRegisterFetchTask.getTaskId(), subscriberRegisterFetchTask, subscriberRegisterFetchTask.getExpiryTime()); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/UnPublishDataTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/UnPublishDataTaskListener.java new file mode 100644 index 000000000..d8a4fa35d --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/UnPublishDataTaskListener.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.listener; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.server.session.scheduler.task.UnPublishDataTask; +import com.alipay.sofa.registry.task.batcher.TaskDispatcher; +import com.alipay.sofa.registry.task.batcher.TaskDispatchers; +import com.alipay.sofa.registry.task.batcher.TaskProcessor; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListener; + +/** + * + * @author kezhu.wukz + * @version $Id: UnPublishDataTaskListener.java, v 0.1 2019-06-17 12:02 kezhu.wukz Exp $ + */ +public class UnPublishDataTaskListener implements TaskListener { + /** + * transfer data to DataNode + */ + @Autowired + private DataNodeService dataNodeService; + + private TaskDispatcher singleTaskDispatcher; + + @Autowired + private TaskProcessor dataNodeSingleTaskProcessor; + + @PostConstruct + public void init() { + singleTaskDispatcher = TaskDispatchers.createSingleTaskDispatcher( + TaskDispatchers.getDispatcherName(TaskType.UN_PUBLISH_DATA_TASK.getName()), 100000, 32, + 1000, 1000, dataNodeSingleTaskProcessor); + } + + @Override + public TaskType support() { + return TaskType.UN_PUBLISH_DATA_TASK; + } + + @Override + public void handleEvent(TaskEvent event) { + SessionTask unPublishDataTask = new UnPublishDataTask(dataNodeService); + + unPublishDataTask.setTaskEvent(event); + + singleTaskDispatcher.dispatch(unPublishDataTask.getTaskId(), unPublishDataTask, + unPublishDataTask.getExpiryTime()); + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/WatcherRegisterFetchTaskListener.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/WatcherRegisterFetchTaskListener.java index 68e4cd6f9..45e14542c 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/WatcherRegisterFetchTaskListener.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/listener/WatcherRegisterFetchTaskListener.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.listener; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.MetaNodeService; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; @@ -27,7 +29,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListener; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -37,23 +38,23 @@ public class WatcherRegisterFetchTaskListener implements TaskListener { @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; /** * trigger push client process */ @Autowired - private TaskListenerManager taskListenerManager; + private TaskListenerManager taskListenerManager; /** * MetaNode service */ @Autowired - private MetaNodeService metaNodeService; + private MetaNodeService metaNodeService; - private TaskDispatcher singleTaskDispatcher; + private volatile TaskDispatcher singleTaskDispatcher; - private TaskProcessor dataNodeSingleTaskProcessor; + private TaskProcessor dataNodeSingleTaskProcessor; public WatcherRegisterFetchTaskListener(TaskProcessor dataNodeSingleTaskProcessor) { this.dataNodeSingleTaskProcessor = dataNodeSingleTaskProcessor; @@ -61,15 +62,21 @@ public WatcherRegisterFetchTaskListener(TaskProcessor dataNodeSingleTaskProcesso public TaskDispatcher getSingleTaskDispatcher() { if (singleTaskDispatcher == null) { - singleTaskDispatcher = TaskDispatchers.createDefaultSingleTaskDispatcher( - TaskType.WATCHER_REGISTER_FETCH_TASK.getName(), dataNodeSingleTaskProcessor); + synchronized (this) { + if (singleTaskDispatcher == null) { + singleTaskDispatcher = TaskDispatchers + .createDefaultSingleTaskDispatcher( + TaskType.WATCHER_REGISTER_FETCH_TASK.getName(), + dataNodeSingleTaskProcessor); + } + } } return singleTaskDispatcher; } @Override - public boolean support(TaskEvent event) { - return TaskType.WATCHER_REGISTER_FETCH_TASK.equals(event.getTaskType()); + public TaskType support() { + return TaskType.WATCHER_REGISTER_FETCH_TASK; } @Override diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/AbstractNodeManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/AbstractNodeManager.java index 274aa4dc4..ea956a588 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/AbstractNodeManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/AbstractNodeManager.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.session.node; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.metaserver.GetNodesRequest; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; @@ -28,14 +37,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.util.VersionsMapUtils; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * @@ -148,9 +149,9 @@ public URL getRequestUrl() { } } catch (RequestException e) { - LOGGER.error("NodeManager get all dataCenter nodes error! " + e.getRequestMessage(), e); + LOGGER.error("NodeManager get all dataCenter nodes error! " + e.getMessage(), e); throw new RuntimeException("NodeManager get all dataCenter nodes error! " - + e.getRequestMessage(), e); + + e.getMessage(), e); } return nodeChangeResult; diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/DataNodeManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/DataNodeManager.java index 894346efa..d62baaff7 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/DataNodeManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/DataNodeManager.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.node; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.DataNode; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; @@ -23,7 +25,6 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -44,7 +45,6 @@ public class DataNodeManager extends AbstractNodeManager { public DataNode getNode(String dataInfoId) { DataNode dataNode = consistentHash.getNodeFor(dataInfoId); if (dataNode == null) { - LOGGER.error("calculate data node error!,dataInfoId={}", dataInfoId); throw new RuntimeException("DataNodeManager calculate data node error!,dataInfoId=" + dataInfoId); } @@ -70,7 +70,7 @@ public NodeType getNodeType() { } @Override - public void reNewNode() { + public void renewNode() { } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/MetaNodeManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/MetaNodeManager.java index 2a2095096..56117598a 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/MetaNodeManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/MetaNodeManager.java @@ -16,11 +16,11 @@ */ package com.alipay.sofa.registry.server.session.node; +import java.util.Collection; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.MetaNode; -import java.util.Collection; - /** * * @author shangyu.wh @@ -44,6 +44,6 @@ public Collection getDataCenters() { } @Override - public void reNewNode() { + public void renewNode() { } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/NodeManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/NodeManager.java index 59b47879a..c3c0ebc74 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/NodeManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/NodeManager.java @@ -16,13 +16,13 @@ */ package com.alipay.sofa.registry.server.session.node; +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import java.util.Collection; -import java.util.concurrent.ConcurrentHashMap; - /** * * @author shangyu.wh @@ -40,7 +40,7 @@ public interface NodeManager { Collection getDataCenters(); - void reNewNode(); + void renewNode(); NodeChangeResult getAllDataCenterNodes(); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/RaftClientManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/RaftClientManager.java index 4eb4c370c..1df88bdbb 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/RaftClientManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/RaftClientManager.java @@ -24,12 +24,15 @@ import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.server.session.bootstrap.CommonConfig; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import org.springframework.beans.factory.annotation.Autowired; +import sun.nio.ch.ThreadPool; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -48,6 +51,9 @@ public class RaftClientManager { @Autowired private CommonConfig commonConfig; + @Autowired + private ExecutorManager executorManager; + private RaftClient raftClient; private AtomicBoolean clientStart = new AtomicBoolean(false); @@ -58,7 +64,8 @@ public void startRaftClient() { try { if (clientStart.compareAndSet(false, true)) { String serverConf = getServerConfig(); - raftClient = new RaftClient(getGroup(), serverConf); + raftClient = new RaftClient(getGroup(), serverConf, + executorManager.getDefaultRequestExecutor()); raftClient.start(); } } catch (Exception e) { @@ -72,7 +79,8 @@ private String getServerConfig() { String ret = ""; Set ips = getMetaIp(); if (ips != null && !ips.isEmpty()) { - ret = ips.stream().map(ip -> ip + ":" + ValueConstants.RAFT_SERVER_PORT).collect(Collectors.joining(",")); + ret = ips.stream().map(ip -> ip + ":" + ValueConstants.RAFT_SERVER_PORT) + .collect(Collectors.joining(",")); } if (ret.isEmpty()) { throw new IllegalArgumentException("Init raft server config error!"); @@ -94,7 +102,8 @@ public Set getMetaIp() { metas.forEach(domain -> { String ip = NetUtil.getIPAddressFromDomain(domain); if (ip == null) { - throw new RuntimeException("Node config convert domain {" + domain + "} error!"); + throw new RuntimeException( + "Node config convert domain {" + domain + "} error!"); } metaIps.add(ip); }); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/SessionNodeManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/SessionNodeManager.java index 4c50cfa2e..b08d4a7bc 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/SessionNodeManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/SessionNodeManager.java @@ -16,9 +16,11 @@ */ package com.alipay.sofa.registry.server.session.node; +import java.util.*; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.metaserver.NodeChangeResult; -import com.alipay.sofa.registry.common.model.metaserver.ReNewNodesRequest; +import com.alipay.sofa.registry.common.model.metaserver.RenewNodesRequest; import com.alipay.sofa.registry.common.model.metaserver.SessionNode; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; @@ -27,10 +29,6 @@ import com.alipay.sofa.registry.remoting.exchange.RequestException; import com.alipay.sofa.registry.remoting.exchange.message.Request; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - /** * * @author shangyu.wh @@ -71,18 +69,18 @@ public List getZoneServerList(String zonename) { } @Override - public void reNewNode() { + public void renewNode() { try { - Request reNewNodesRequestRequest = new Request() { + Request renewNodesRequestRequest = new Request() { @Override - public ReNewNodesRequest getRequestBody() { + public RenewNodesRequest getRequestBody() { URL clientUrl = new URL(NetUtil.getLocalAddress().getHostAddress(), 0); SessionNode sessionNode = new SessionNode(clientUrl, sessionServerConfig.getSessionServerRegion()); - return new ReNewNodesRequest(sessionNode); + return new RenewNodesRequest(sessionNode); } @Override @@ -92,11 +90,9 @@ public URL getRequestUrl() { } }; - metaNodeExchanger.request(reNewNodesRequestRequest); + metaNodeExchanger.request(renewNodesRequestRequest); } catch (RequestException e) { - LOGGER.error("SessionNodeManager reNew node error! " + e.getRequestMessage(), e); - throw new RuntimeException("SessionNodeManager reNew node error! " - + e.getRequestMessage(), e); + throw new RuntimeException("SessionNodeManager renew node error! " + e.getMessage(), e); } } @@ -118,4 +114,4 @@ public void updateNodes(NodeChangeResult nodeChangeResult) { write.unlock(); } } -} \ No newline at end of file +} diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/processor/DataNodeSingleTaskProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/processor/DataNodeSingleTaskProcessor.java index 74d6c9c43..f3e376b90 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/processor/DataNodeSingleTaskProcessor.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/processor/DataNodeSingleTaskProcessor.java @@ -16,14 +16,15 @@ */ package com.alipay.sofa.registry.server.session.node.processor; +import java.util.List; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.scheduler.task.RenewDatumTask; import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; import com.alipay.sofa.registry.task.Retryable; import com.alipay.sofa.registry.task.batcher.TaskProcessor; -import java.util.List; - /** * * @author shangyu.wh @@ -37,9 +38,15 @@ public class DataNodeSingleTaskProcessor implements TaskProcessor { @Override public ProcessingResult process(SessionTask task) { try { - LOGGER.info("execute " + task); + if (!(task instanceof RenewDatumTask)) { + LOGGER.info("execute " + task); + } + task.execute(); - LOGGER.info("end " + task); + + if (!(task instanceof RenewDatumTask)) { + LOGGER.info("end " + task); + } return ProcessingResult.Success; } catch (Throwable throwable) { LOGGER.error("Data node SingleTask Process error! Task:" + task, throwable); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/ClientNodeServiceImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/ClientNodeServiceImpl.java index 7ab6333cf..788c6b844 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/ClientNodeServiceImpl.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/ClientNodeServiceImpl.java @@ -16,12 +16,13 @@ */ package com.alipay.sofa.registry.server.session.node.service; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.remoting.CallbackHandler; import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; import com.alipay.sofa.registry.remoting.exchange.RequestException; import com.alipay.sofa.registry.remoting.exchange.message.Request; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -57,8 +58,7 @@ public CallbackHandler getCallBackHandler() { clientNodeExchanger.request(request); } catch (RequestException e) { - throw new RuntimeException("Push data to client node error !" + e.getRequestMessage(), - e); + throw new RuntimeException("Push data to client node error !" + e.getMessage(), e); } } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeService.java index df1180ba8..030265299 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeService.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeService.java @@ -16,15 +16,16 @@ */ package com.alipay.sofa.registry.server.session.node.service; -import com.alipay.sofa.registry.common.model.dataserver.Datum; -import com.alipay.sofa.registry.common.model.dataserver.SessionServerRegisterRequest; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.common.model.store.URL; - import java.util.Collection; import java.util.List; import java.util.Map; +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.URL; + /** * * @author shangyu.wh @@ -89,12 +90,13 @@ public interface DataNodeService { Map getDatumMap(String dataInfoId, String dataCenterId); /** - * register session process id when connect to data node - * process id see SessionProcessIdGenerator - * @param sessionServerRegisterRequest - * @param dataUrl + * check publisher digest same as session current store,and renew the lastUpdateTime of this connectId + */ + Boolean renewDatum(RenewDatumRequest renewDatumRequest); + + /** + * Correct the publishers information of this connectId on dataServer */ - void registerSessionProcessId(SessionServerRegisterRequest sessionServerRegisterRequest, - URL dataUrl); + void sendDatumSnapshot(DatumSnapshotRequest datumSnapshotRequest); } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeServiceImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeServiceImpl.java index 4337f2432..4b203f5bb 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeServiceImpl.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/DataNodeServiceImpl.java @@ -16,15 +16,27 @@ */ package com.alipay.sofa.registry.server.session.node.service; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; import com.alipay.sofa.registry.common.model.dataserver.ClientOffRequest; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.dataserver.GetDataRequest; import com.alipay.sofa.registry.common.model.dataserver.GetDataVersionRequest; import com.alipay.sofa.registry.common.model.dataserver.PublishDataRequest; -import com.alipay.sofa.registry.common.model.dataserver.SessionServerRegisterRequest; import com.alipay.sofa.registry.common.model.dataserver.UnPublishDataRequest; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.URL; @@ -40,14 +52,6 @@ import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; /** * @@ -69,13 +73,16 @@ public class DataNodeServiceImpl implements DataNodeService { private AsyncHashedWheelTimer asyncHashedWheelTimer; - public DataNodeServiceImpl() { + @PostConstruct + public void init() { ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); threadFactoryBuilder.setDaemon(true); asyncHashedWheelTimer = new AsyncHashedWheelTimer(threadFactoryBuilder.setNameFormat( - "Registry-DataNodeServiceImpl-WheelTimer").build(), 100, TimeUnit.MILLISECONDS, 1024, - threadFactoryBuilder.setNameFormat("Registry-DataNodeServiceImpl-WheelExecutor-%d") - .build(), new TaskFailedCallback() { + "Registry-DataNodeServiceImpl-Retry-WheelTimer").build(), 100, TimeUnit.MILLISECONDS, + 1024, sessionServerConfig.getDataNodeRetryExecutorThreadSize(), + sessionServerConfig.getDataNodeRetryExecutorQueueSize(), threadFactoryBuilder + .setNameFormat("Registry-DataNodeServiceImpl-Retry-WheelExecutor-%d").build(), + new TaskFailedCallback() { @Override public void executionRejected(Throwable e) { LOGGER.error("executionRejected: " + e.getMessage(), e); @@ -90,98 +97,78 @@ public void executionFailed(Throwable e) { @Override public void register(final Publisher publisher) { - + String bizName = "PublishData"; + Request request = buildPublishDataRequest(publisher); try { + sendRequest(bizName, request); + } catch (RequestException e) { + doRetryAsync(bizName, request, e, sessionServerConfig.getPublishDataTaskRetryTimes(), + sessionServerConfig.getPublishDataTaskRetryFirstDelay(), + sessionServerConfig.getPublishDataTaskRetryIncrementDelay()); + } + } - Request publisherRequest = new Request() { - - private URL url; - - @Override - public PublishDataRequest getRequestBody() { - PublishDataRequest publishDataRequest = new PublishDataRequest(); - publishDataRequest.setPublisher(publisher); - publishDataRequest.setSessionServerProcessId(SessionProcessIdGenerator - .getSessionProcessId()); - return publishDataRequest; - } - - @Override - public URL getRequestUrl() { - if (url == null) { - url = getUrl(publisher.getDataInfoId()); - } - return url; - } - }; + private Request buildPublishDataRequest(Publisher publisher) { + return new Request() { + private AtomicInteger retryTimes = new AtomicInteger(); + + @Override + public PublishDataRequest getRequestBody() { + PublishDataRequest publishDataRequest = new PublishDataRequest(); + publishDataRequest.setPublisher(publisher); + publishDataRequest.setSessionServerProcessId(SessionProcessIdGenerator + .getSessionProcessId()); + return publishDataRequest; + } - Response response = dataNodeExchanger.request(publisherRequest); + @Override + public URL getRequestUrl() { + return getUrl(publisher.getDataInfoId()); + } - Object result = response.getResult(); - if (result instanceof CommonResponse) { - CommonResponse commonResponse = (CommonResponse) result; - if (!commonResponse.isSuccess()) { - LOGGER.error( - "PublishDataRequest get server response failed!target url:{},message:{}", - publisherRequest.getRequestUrl(), commonResponse.getMessage()); - throw new RuntimeException( - "PublishDataRequest get server response failed! msg:" - + commonResponse.getMessage()); - } + @Override + public AtomicInteger getRetryTimes() { + return retryTimes; } - } catch (RequestException e) { - LOGGER.error("DataNodeService register new publisher error! " + e.getRequestMessage(), - e); - throw new RuntimeException("DataNodeService register new publisher error! " - + e.getRequestMessage(), e); - } + }; } @Override public void unregister(final Publisher publisher) { + String bizName = "UnPublishData"; + Request request = buildUnPublishDataRequest(publisher); try { - Request unPublishRequest = new Request() { - - private URL url; - - @Override - public UnPublishDataRequest getRequestBody() { - UnPublishDataRequest unPublishDataRequest = new UnPublishDataRequest(); - unPublishDataRequest.setDataInfoId(publisher.getDataInfoId()); - unPublishDataRequest.setRegisterId(publisher.getRegisterId()); - unPublishDataRequest.setRegisterTimestamp(publisher.getRegisterTimestamp()); - return unPublishDataRequest; - } + sendRequest(bizName, request); + } catch (RequestException e) { + doRetryAsync(bizName, request, e, sessionServerConfig.getUnPublishDataTaskRetryTimes(), + sessionServerConfig.getUnPublishDataTaskRetryFirstDelay(), + sessionServerConfig.getUnPublishDataTaskRetryIncrementDelay()); + } + } - @Override - public URL getRequestUrl() { - if (url == null) { - url = getUrl(publisher.getDataInfoId()); - } - return url; - } - }; + private Request buildUnPublishDataRequest(Publisher publisher) { + return new Request() { - Response response = dataNodeExchanger.request(unPublishRequest); + private AtomicInteger retryTimes = new AtomicInteger(); - Object result = response.getResult(); - if (result instanceof CommonResponse) { - CommonResponse commonResponse = (CommonResponse) result; - if (!commonResponse.isSuccess()) { - LOGGER.error( - "UnPublishRequest get server response failed!target url:{},message:{}", - unPublishRequest.getRequestUrl(), commonResponse.getMessage()); - throw new RuntimeException("UnPublishRequest get server response failed! msg:" - + commonResponse.getMessage()); - } + @Override + public UnPublishDataRequest getRequestBody() { + UnPublishDataRequest unPublishDataRequest = new UnPublishDataRequest( + publisher.getDataInfoId(), publisher.getRegisterId(), + publisher.getRegisterTimestamp()); + return unPublishDataRequest; } - } catch (RequestException e) { - LOGGER.error("Unregister publisher to data node error! " + e.getRequestMessage(), e); - throw new RuntimeException("Unregister publisher to data node error! " - + e.getRequestMessage(), e); - } + @Override + public URL getRequestUrl() { + return getUrl(publisher.getDataInfoId()); + } + @Override + public AtomicInteger getRetryTimes() { + return retryTimes; + } + }; } @Override @@ -190,147 +177,55 @@ public void clientOff(List connectIds) { return; } //get all local dataCenter data node + String bizName = "ClientOff"; Collection nodes = dataNodeManager.getDataCenterNodes(); if (nodes != null && nodes.size() > 0) { - for (Node node : nodes) { - Request clientOffRequestRequest = new Request() { - - private AtomicInteger retryTimes = new AtomicInteger(); - - @Override - public ClientOffRequest getRequestBody() { - ClientOffRequest clientOffRequest = new ClientOffRequest(); - clientOffRequest.setHosts(connectIds); - clientOffRequest.setGmtOccur(System.currentTimeMillis()); - return clientOffRequest; - } - - @Override - public URL getRequestUrl() { - return new URL(node.getNodeUrl().getIpAddress(), - sessionServerConfig.getDataServerPort()); - } - - @Override - public AtomicInteger getRetryTimes() { - return retryTimes; - } - }; + Request request = buildClientOffRequest(connectIds, node); try { + sendRequest(bizName, request); + } catch (RequestException e) { + doRetryAsync(bizName, request, e, + sessionServerConfig.getCancelDataTaskRetryTimes(), + sessionServerConfig.getCancelDataTaskRetryFirstDelay(), + sessionServerConfig.getCancelDataTaskRetryIncrementDelay()); - Response response = dataNodeExchanger.request(clientOffRequestRequest); - Object result = response.getResult(); - if (result instanceof CommonResponse) { - CommonResponse commonResponse = (CommonResponse) result; - if (!commonResponse.isSuccess()) { - LOGGER - .error( - "ClientOff RequestRequest get response failed!target url:{},message:{}", - node.getNodeUrl(), commonResponse.getMessage()); - throw new RuntimeException( - "ClientOff RequestRequest get response failed! msg:" - + commonResponse.getMessage()); - } - } else { - LOGGER - .error( - "ClientOff Request has not get response or response type illegal!url:{}", - node.getNodeUrl()); - throw new RuntimeException( - "ClientOff Request has not get response or response type illegal!"); - } - } catch (Exception e) { - LOGGER.error("Client Off request error! ", e); - clientOffRetry(clientOffRequestRequest); } - } } } - private void clientOffRetry(Request clientOffRequestRequest) { - - URL url = clientOffRequestRequest.getRequestUrl(); + private Request buildClientOffRequest(List connectIds, Node node) { + return new Request() { - int retryTimes = clientOffRequestRequest.getRetryTimes().incrementAndGet(); + private AtomicInteger retryTimes = new AtomicInteger(); - if (retryTimes <= sessionServerConfig.getCancelDataTaskRetryTimes()) { - asyncHashedWheelTimer.newTimeout(timeout -> { - try { - Response response = dataNodeExchanger.request(clientOffRequestRequest); - Object result = response.getResult(); - if (result instanceof CommonResponse) { - CommonResponse commonResponse = (CommonResponse) result; - if (!commonResponse.isSuccess()) { - LOGGER.error( - "ClientOff retry RequestRequest get response failed!retryTimes={},target url:{},message:{}", - retryTimes, - url, commonResponse.getMessage()); - throw new RuntimeException( - "ClientOff retry RequestRequest get response failed! msg:" + - commonResponse.getMessage()); - } - } else { - LOGGER.error( - "ClientOff retry Request has not get response or response type illegal!retryTimes={},url:{}", - retryTimes, url); - throw new RuntimeException( - "ClientOff retry Request has not get response or response type illegal!"); - } - } catch (Exception e) { - clientOffRetry(clientOffRequestRequest); - } - }, getBlockTime(retryTimes), TimeUnit.MILLISECONDS); - } else { - LOGGER.error("ClientOff retryTimes have exceeded! stop retry! retryTimes={}, url={}, request={}", - retryTimes, url, clientOffRequestRequest.getRequestBody()); - } - } - - private long getBlockTime(int retry) { - long initialSleepTime = TimeUnit.MILLISECONDS.toMillis(sessionServerConfig - .getCancelDataTaskRetryFirstDelay()); - long increment = TimeUnit.MILLISECONDS.toMillis(sessionServerConfig - .getCancelDataTaskRetryIncrementDelay()); - long result = initialSleepTime + (increment * (retry - 1)); - return result >= 0L ? result : 0L; - } - - @Override - public void registerSessionProcessId(final SessionServerRegisterRequest sessionServerRegisterRequest, - final URL dataUrl) { - - try { - - Request request = new Request() { - - @Override - public SessionServerRegisterRequest getRequestBody() { - return sessionServerRegisterRequest; - } + @Override + public ClientOffRequest getRequestBody() { + ClientOffRequest clientOffRequest = new ClientOffRequest(); + clientOffRequest.setHosts(connectIds); + clientOffRequest.setGmtOccur(System.currentTimeMillis()); + return clientOffRequest; + } - @Override - public URL getRequestUrl() { - return dataUrl; - } - }; + @Override + public URL getRequestUrl() { + return new URL(node.getNodeUrl().getIpAddress(), + sessionServerConfig.getDataServerPort()); + } - dataNodeExchanger.request(request); - } catch (RequestException e) { - LOGGER.error("DataNodeService register processId error! " + e.getRequestMessage(), e); - throw new RuntimeException("DataNodeService register processId error! " - + e.getRequestMessage(), e); - } + @Override + public AtomicInteger getRetryTimes() { + return retryTimes; + } + }; } @Override public Map> fetchDataVersion(URL dataNodeUrl, Collection dataInfoIdList) { - Map> map = new HashMap<>(); try { - Request getDataVersionRequestRequest = new Request() { @Override public GetDataVersionRequest getRequestBody() { @@ -346,33 +241,22 @@ public URL getRequestUrl() { }; Response response = dataNodeExchanger.request(getDataVersionRequestRequest); - Object result = response.getResult(); - if (result instanceof GenericResponse) { - GenericResponse genericResponse = (GenericResponse) result; - if (genericResponse.isSuccess()) { - map = (Map>) genericResponse.getData(); - if (map.isEmpty()) { - LOGGER - .warn( - "GetDataVersionRequestRequest get response contains no data!target data Node url:{} about dataInfoIds size:{}", - dataNodeUrl.getAddressString(), dataInfoIdList.size()); - } - } else { - LOGGER.error("fetchDataVersion has not get fail response!msg:{}", - genericResponse.getMessage()); - throw new RuntimeException("fetchDataVersion has not get fail response! msg:" - + genericResponse.getMessage()); + GenericResponse genericResponse = (GenericResponse) result; + if (genericResponse.isSuccess()) { + map = (Map>) genericResponse.getData(); + if (map.isEmpty()) { + LOGGER + .warn( + "GetDataVersionRequestRequest get response contains no data!target data Node url:{} about dataInfoIds size:{}", + dataNodeUrl.getAddressString(), dataInfoIdList.size()); } } else { - LOGGER - .error("GetDataVersionRequestRequest has not get response or response type illegal!"); + throw new RuntimeException("fetchDataVersion has not get fail response! msg:" + + genericResponse.getMessage()); } - } catch (RequestException e) { - LOGGER.error("Fetch data Version request error! " + e.getRequestMessage(), e); - throw new RuntimeException( - "Fetch data Version request error! " + e.getRequestMessage(), e); + throw new RuntimeException("Fetch data Version request error! " + e.getMessage(), e); } return map; @@ -398,12 +282,12 @@ private Map getDatumMap(String dataInfoId) { return getDatumMap(dataInfoId, null); } - @Override public Map getDatumMap(String dataInfoId, String dataCenterId) { + @Override + public Map getDatumMap(String dataInfoId, String dataCenterId) { Map map; try { - GetDataRequest getDataRequest = new GetDataRequest(); //dataCenter null means all dataCenters @@ -424,46 +308,150 @@ public GetDataRequest getRequestBody() { public URL getRequestUrl() { return getUrl(dataInfoId); } + + @Override + public Integer getTimeout() { + return sessionServerConfig.getDataNodeExchangeForFetchDatumTimeOut(); + } }; Response response = dataNodeExchanger.request(getDataRequestStringRequest); Object result = response.getResult(); - if (result instanceof GenericResponse) { - GenericResponse genericResponse = (GenericResponse) result; - if (genericResponse.isSuccess()) { - map = (Map) genericResponse.getData(); - if (map == null || map.isEmpty()) { - LOGGER.warn("GetDataRequest get response contains no datum!"); - } else { - map.forEach((dataCenter, datum) -> Datum.processDatum(datum)); - } + GenericResponse genericResponse = (GenericResponse) result; + if (genericResponse.isSuccess()) { + map = (Map) genericResponse.getData(); + if (map == null || map.isEmpty()) { + LOGGER.warn("GetDataRequest get response contains no datum!dataInfoId={}", dataInfoId); } else { - LOGGER.error("GetDataRequest has not get fail response!msg:{}", genericResponse.getMessage()); - throw new RuntimeException("GetDataRequest has not get fail response! msg:" + - genericResponse.getMessage()); + map.forEach((dataCenter, datum) -> Datum.internDatum(datum)); } } else { - LOGGER.error("GetDataRequest has not get response or response type illegal!"); - throw new RuntimeException("GetDataRequest has not get response or response type illegal!"); + throw new RuntimeException( + String.format("GetDataRequest has got fail response!dataInfoId:%s msg:%s", dataInfoId, + genericResponse.getMessage())); } } catch (RequestException e) { - LOGGER.error("Get data request to data node error! " + e.getRequestMessage(), e); throw new RuntimeException( - "Get data request to data node error! " + e.getRequestMessage(), e); + String.format("Get data request to data node error!dataInfoId:%s msg:%s ", dataInfoId, + e.getMessage()), e); } return map; } - private URL getUrl(String dataInfoId) { + @Override + public Boolean renewDatum(RenewDatumRequest renewDatumRequest) { + Request request = buildRenewDatumRequest(renewDatumRequest); + try { + GenericResponse genericResponse = (GenericResponse) sendRequest("RenewDatum", request); + return (Boolean) genericResponse.getData(); + } catch (RequestException e) { + throw new RuntimeException(e.getMessage(), e); + } + } - Node dataNode = dataNodeManager.getNode(dataInfoId); - if (dataNode != null) { - //meta push data node has not port - String dataIp = dataNode.getNodeUrl().getIpAddress(); - return new URL(dataIp, sessionServerConfig.getDataServerPort()); + private Request buildRenewDatumRequest(RenewDatumRequest renewDatumRequest) { + return new Request() { + private AtomicInteger retryTimes = new AtomicInteger(); + + @Override + public RenewDatumRequest getRequestBody() { + return renewDatumRequest; + } + + @Override + public URL getRequestUrl() { + return new URL(renewDatumRequest.getDataServerIP(), + sessionServerConfig.getDataServerPort()); + } + + @Override + public AtomicInteger getRetryTimes() { + return retryTimes; + } + }; + } + + @Override + public void sendDatumSnapshot(DatumSnapshotRequest datumSnapshotRequest) { + String bizName = "DatumSnapshot"; + Request request = buildDatumSnapshotRequest(datumSnapshotRequest); + try { + sendRequest(bizName, request); + } catch (RequestException e) { + doRetryAsync(bizName, request, e, sessionServerConfig.getDatumSnapshotTaskRetryTimes(), + sessionServerConfig.getDatumSnapshotTaskRetryFirstDelay(), + sessionServerConfig.getDatumSnapshotTaskRetryIncrementDelay()); } - return null; + + } + + private Request buildDatumSnapshotRequest(DatumSnapshotRequest datumSnapshotRequest) { + return new Request() { + private AtomicInteger retryTimes = new AtomicInteger(); + + @Override + public DatumSnapshotRequest getRequestBody() { + return datumSnapshotRequest; + } + + @Override + public URL getRequestUrl() { + return new URL(datumSnapshotRequest.getDataServerIp(), + sessionServerConfig.getDataServerPort()); + } + + @Override + public AtomicInteger getRetryTimes() { + return retryTimes; + } + }; + } + + private CommonResponse sendRequest(String bizName, Request request) throws RequestException { + Response response = dataNodeExchanger.request(request); + Object result = response.getResult(); + CommonResponse commonResponse = (CommonResponse) result; + if (!commonResponse.isSuccess()) { + throw new RuntimeException(String.format( + "[%s] response not success, failed! target url: %s, request: %s, message: %s", + bizName, request.getRequestUrl(), request.getRequestBody(), + commonResponse.getMessage())); + } + return commonResponse; + } + + private void doRetryAsync(String bizName, Request request, Exception e, int maxRetryTimes, long firstDelay, + long incrementDelay) { + int retryTimes = request.getRetryTimes().incrementAndGet(); + if (retryTimes <= maxRetryTimes) { + LOGGER.warn("{} failed, will retry again, retryTimes: {}, msg: {}", bizName, retryTimes, e.getMessage()); + asyncHashedWheelTimer.newTimeout(timeout -> { + try { + sendRequest(bizName, request); + } catch (RequestException ex) { + doRetryAsync(bizName, request, ex, maxRetryTimes, firstDelay, incrementDelay); + } + }, getDelayTime(retryTimes, firstDelay, incrementDelay), TimeUnit.MILLISECONDS); + } else { + LOGGER.error(String.format( + "%s failed, retryTimes have exceeded! stop retry! retryTimes: %s, url: %s, request: %s, msg: %s", + bizName, (retryTimes - 1), request.getRequestUrl(), request.getRequestBody(), e.getMessage()), e); + } + } + + private long getDelayTime(int retry, long firstDelay, long incrementDelay) { + long initialSleepTime = TimeUnit.MILLISECONDS.toMillis(firstDelay); + long increment = TimeUnit.MILLISECONDS.toMillis(incrementDelay); + long result = initialSleepTime + (increment * (retry - 1)); + return result >= 0L ? result : 0L; + } + + private URL getUrl(String dataInfoId) { + Node dataNode = dataNodeManager.getNode(dataInfoId); + //meta push data node has not port + String dataIp = dataNode.getNodeUrl().getIpAddress(); + return new URL(dataIp, sessionServerConfig.getDataServerPort()); } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/MetaNodeServiceImpl.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/MetaNodeServiceImpl.java index 85eef1f9a..eaceb1e09 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/MetaNodeServiceImpl.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/node/service/MetaNodeServiceImpl.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.node.service; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.metaserver.FetchProvideDataRequest; import com.alipay.sofa.registry.common.model.metaserver.ProvideData; import com.alipay.sofa.registry.common.model.store.URL; @@ -28,7 +30,6 @@ import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.RaftClientManager; import com.alipay.sofa.registry.server.session.node.SessionNodeManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -78,8 +79,8 @@ public URL getRequestUrl() { throw new RuntimeException("MetaNodeService fetch null provider data!"); } } catch (RequestException e) { - LOGGER.error("fetch provider data error! " + e.getRequestMessage(), e); - throw new RuntimeException("fetch provider data error! " + e.getRequestMessage(), e); + LOGGER.error("fetch provider data error! " + e.getMessage(), e); + throw new RuntimeException("fetch provider data error! " + e.getMessage(), e); } } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessor.java new file mode 100644 index 000000000..91566fa98 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.provideData; + +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; + +/** + * + * @author shangyu.wh + * @version 1.0: ProvideDataProcessor.java, v 0.1 2019-10-09 17:26 shangyu.wh Exp $ + */ +public interface ProvideDataProcessor { + + void changeDataProcess(ProvideData provideData); + + void fetchDataProcess(ProvideData provideData); + + boolean support(ProvideData provideData); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessorManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessorManager.java new file mode 100644 index 000000000..c68dafc46 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/ProvideDataProcessorManager.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.provideData; + +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * + * @author shangyu.wh + * @version 1.0: ProvideDataProcessorManager.java, v 0.1 2019-10-09 17:39 shangyu.wh Exp $ + */ +public class ProvideDataProcessorManager implements ProvideDataProcessor { + + private Collection provideDataProcessors = new ArrayList<>(); + + public void addProvideDataProcessor(ProvideDataProcessor provideDataProcessor) { + provideDataProcessors.add(provideDataProcessor); + } + + @Override + public void changeDataProcess(ProvideData provideData) { + for (ProvideDataProcessor provideDataProcessor : provideDataProcessors) { + if (provideDataProcessor.support(provideData)) { + provideDataProcessor.changeDataProcess(provideData); + } + } + } + + @Override + public void fetchDataProcess(ProvideData provideData) { + for (ProvideDataProcessor provideDataProcessor : provideDataProcessors) { + if (provideDataProcessor.support(provideData)) { + provideDataProcessor.fetchDataProcess(provideData); + } + } + } + + @Override + public boolean support(ProvideData provideData) { + return false; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/BlackListProvideDataProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/BlackListProvideDataProcessor.java new file mode 100644 index 000000000..60fae4f7f --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/BlackListProvideDataProcessor.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.provideData.processor; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistConstants; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistManager; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.session.registry.Registry; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * + * @author shangyu.wh + * @version 1.0: BlackListProvideDataProcessor.java, v 0.1 2019-10-09 20:21 shangyu.wh Exp $ + */ +public class BlackListProvideDataProcessor implements ProvideDataProcessor { + + private static final Logger LOGGER = LoggerFactory + .getLogger(BlackListProvideDataProcessor.class); + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + private Registry sessionRegistry; + + @Autowired + private Exchange boltExchange; + + @Autowired + private BlacklistManager blacklistManager; + + @Override + public void changeDataProcess(ProvideData provideData) { + //black list data + if (provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER.info("Fetch session blacklist no data existed,current config not change!"); + return; + } + String data = (String) provideData.getProvideData().getObject(); + if (data != null) { + Map>> blacklistConfigMap = blacklistManager + .convertBlacklistConfig(data); + clientOffBlackIp(blacklistConfigMap); + LOGGER.info("Fetch session blacklist data switch {} success!", data); + } else { + LOGGER.info("Fetch session blacklist data null,current config not change!"); + } + return; + } + + private void clientOffBlackIp(Map>> blacklistConfigMap) { + + if (blacklistConfigMap != null) { + Set ipSet = new HashSet(); + + for (Map.Entry>> configEntry : blacklistConfigMap + .entrySet()) { + if (BlacklistConstants.FORBIDDEN_PUB.equals(configEntry.getKey()) + || BlacklistConstants.FORBIDDEN_SUB_BY_PREFIX.equals(configEntry.getKey())) { + Map> typeMap = configEntry.getValue(); + if (typeMap != null) { + for (Map.Entry> typeEntry : typeMap.entrySet()) { + if (BlacklistConstants.IP_FULL.equals(typeEntry.getKey())) { + if (typeEntry.getValue() != null) { + ipSet.addAll(typeEntry.getValue()); + } + } + } + } + } + + } + + sessionRegistry.remove(getIpConnects(ipSet)); + } + } + + public List getIpConnects(Set _ipList) { + + Server sessionServer = boltExchange.getServer(sessionServerConfig.getServerPort()); + + List connections = new ArrayList<>(); + + if (sessionServer != null) { + Collection channels = sessionServer.getChannels(); + for (Channel channel : channels) { + String key = NetUtil.toAddressString(channel.getRemoteAddress()); + String ip = key.substring(0, key.indexOf(":")); + if (_ipList.contains(ip)) { + connections.add(key + ValueConstants.CONNECT_ID_SPLIT + + NetUtil.toAddressString(channel.getLocalAddress())); + } + } + } + + return connections; + } + + @Override + public void fetchDataProcess(ProvideData provideData) { + + } + + @Override + public boolean support(ProvideData provideData) { + return ValueConstants.BLACK_LIST_DATA_ID.equals(provideData.getDataInfoId()); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/RenewSnapshotProvideDataProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/RenewSnapshotProvideDataProcessor.java new file mode 100644 index 000000000..6b829f0a3 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/RenewSnapshotProvideDataProcessor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.provideData.processor; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.session.registry.Registry; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * + * @author shangyu.wh + * @version 1.0: RenewSnapshotProvideDataProcessor.java, v 0.1 2019-10-09 20:30 shangyu.wh Exp $ + */ +public class RenewSnapshotProvideDataProcessor implements ProvideDataProcessor { + + private static final Logger LOGGER = LoggerFactory + .getLogger(RenewSnapshotProvideDataProcessor.class); + + @Autowired + private Registry sessionRegistry; + + @Override + public void changeDataProcess(ProvideData provideData) { + //stop renew switch + if (provideData == null || provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER + .info("Fetch enableDataRenewSnapshot but no data existed, current config not change!"); + return; + } + boolean enableDataRenewSnapshot = Boolean.parseBoolean((String) provideData + .getProvideData().getObject()); + LOGGER.info("Fetch enableDataRenewSnapshot {} success!", enableDataRenewSnapshot); + this.sessionRegistry.setEnableDataRenewSnapshot(enableDataRenewSnapshot); + return; + } + + @Override + public void fetchDataProcess(ProvideData provideData) { + if (provideData == null || provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER + .info("Fetch enableDataRenewSnapshot but no data existed, current config not change!"); + return; + } + boolean enableDataRenewSnapshot = Boolean.parseBoolean((String) provideData + .getProvideData().getObject()); + LOGGER.info("Fetch enableDataRenewSnapshot {} success!", enableDataRenewSnapshot); + this.sessionRegistry.setEnableDataRenewSnapshot(enableDataRenewSnapshot); + } + + @Override + public boolean support(ProvideData provideData) { + return ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT.equals(provideData.getDataInfoId()); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/StopPushProvideDataProcessor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/StopPushProvideDataProcessor.java new file mode 100644 index 000000000..0ab5aa942 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/provideData/processor/StopPushProvideDataProcessor.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.provideData.processor; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.metaserver.ProvideData; +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; +import com.alipay.sofa.registry.server.session.registry.Registry; +import com.alipay.sofa.registry.server.session.scheduler.task.Constant; +import com.alipay.sofa.registry.server.session.store.Interests; +import com.alipay.sofa.registry.server.session.store.ReSubscribers; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * + * @author shangyu.wh + * @version 1.0: StopPushProvideDataProcessor.java, v 0.1 2019-10-09 18:53 shangyu.wh Exp $ + */ +public class StopPushProvideDataProcessor implements ProvideDataProcessor { + + private static final Logger TASK_LOGGER = LoggerFactory.getLogger( + StopPushProvideDataProcessor.class, "[Task]"); + + private static final Logger LOGGER = LoggerFactory + .getLogger(StopPushProvideDataProcessor.class); + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + private Registry sessionRegistry; + + @Autowired + private Interests sessionInterests; + + @Autowired + private TaskListenerManager taskListenerManager; + + @Override + public void changeDataProcess(ProvideData provideData) { + //push stop switch + if (provideData != null) { + if (provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER.info("Fetch session stop push switch no data existed,config not change!"); + return; + } + String data = (String) provideData.getProvideData().getObject(); + LOGGER.info("Fetch session stop push data switch {} success!", data); + + //receive stop push switch off + if (data != null) { + boolean switchData = Boolean.valueOf(data); + boolean ifChange = sessionServerConfig.isStopPushSwitch() != switchData; + sessionServerConfig.setStopPushSwitch(switchData); + if (!switchData) { + //avoid duplicate false receive + if (ifChange) { + fireReSubscriber(); + } + } else { + //stop push and stop fetch data task + sessionServerConfig.setBeginDataFetchTask(false); + } + } else { + LOGGER.error("Fetch session stop push data switch is null!"); + } + return; + } else { + LOGGER.info("Fetch session stop push switch data null,config not change!"); + } + return; + } + + /** + * open push switch to push all reSubscribers + */ + private void fireReSubscriber() { + + //try catch avoid to error cancel beginDataFetchTask switch on + try { + //begin push fire data fetch task first,avoid reSubscriber push duplicate + sessionRegistry.fetchChangDataProcess(); + } catch (Throwable e) { + LOGGER.error("Open push switch first fetch task execute error", e); + } + + try { + //wait 1 MINUTES for dataFetch task evict duplicate subscriber push + TimeUnit.MINUTES.sleep(1); + } catch (InterruptedException e) { + LOGGER.error("Wait for dataFetch Task Interrupted!"); + } + + //fetch task process 1 minutes,can schedule execute fetch task + sessionServerConfig.setBeginDataFetchTask(true); + + if (sessionInterests instanceof ReSubscribers) { + ReSubscribers reSubscriber = (ReSubscribers) sessionInterests; + + Map> reSubscribers = reSubscriber + .getReSubscribers(); + + if (reSubscribers != null && !reSubscribers.isEmpty()) { + reSubscribers.forEach( + (dataInfoId, subscribers) -> fireSubscriberMultiFetchTask(dataInfoId, subscribers.values())); + reSubscriber.clearReSubscribers(); + } + } + } + + private void fireSubscriberMultiFetchTask(String dataInfoId, Collection subscribers) { + //trigger fetch data for subscriber,and push to client node + if (!CollectionUtils.isEmpty(subscribers)) { + TaskEvent taskEvent = new TaskEvent(dataInfoId, TaskType.SUBSCRIBER_MULTI_FETCH_TASK); + taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); + TASK_LOGGER.info("send " + taskEvent.getTaskType() + + " subscribersSize:{},dataInfoId:{}", subscribers.size(), dataInfoId); + taskListenerManager.sendTaskEvent(taskEvent); + } + } + + @Override + public void fetchDataProcess(ProvideData provideData) { + if (provideData.getProvideData() == null + || provideData.getProvideData().getObject() == null) { + LOGGER.info("Fetch session stop push switch no data existed,config not change!"); + return; + } + String data = (String) provideData.getProvideData().getObject(); + sessionServerConfig.setStopPushSwitch(Boolean.valueOf(data)); + if (data != null) { + if (!Boolean.valueOf(data)) { + //stop push init on,then begin fetch data schedule task + sessionServerConfig.setBeginDataFetchTask(true); + } else { + sessionServerConfig.setBeginDataFetchTask(false); + } + } + LOGGER.info("Fetch session stop push data switch {} success!", data); + } + + @Override + public boolean support(ProvideData provideData) { + return ValueConstants.STOP_PUSH_DATA_SWITCH_DATA_ID.equals(provideData.getDataInfoId()); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/Registry.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/Registry.java index 6d4920f14..a0c7c6675 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/Registry.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/Registry.java @@ -16,10 +16,10 @@ */ package com.alipay.sofa.registry.server.session.registry; -import com.alipay.sofa.registry.common.model.store.StoreData; - import java.util.List; +import com.alipay.sofa.registry.common.model.store.StoreData; + /** * * @author shangyu.wh @@ -43,6 +43,15 @@ public interface Registry { */ void cancel(List connectIds); + /** + * remove publisher or subscriber data by client ip address and port(ip:port) + * this function always use in rest api Console manage ,the run mode is standard + * remove subscriber data will push empty datum to some one who has dataInfoId begin with pushEmptyDataDataIdPrefixes config + * + * @param connectIds + */ + void remove(List connectIds); + /** * message mode com.alipay.sofa.registry.client.provider for client node to unregister single subscriber or publisher data * @@ -61,4 +70,18 @@ public interface Registry { * for fetchChangData first invoke */ void fetchChangDataProcess(); + + /** + * Regularly send update Datum with same connectID valid time request service + */ + void renewDatum(String connectId); + + void sendDatumSnapshot(String connectId, String dataServerIp); + + /** + * for scheduler clean no connect client + */ + void cleanClientConnect(); + + void setEnableDataRenewSnapshot(boolean enableDataRenewSnapshot); } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/SessionRegistry.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/SessionRegistry.java index fe40e3a48..5813bab96 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/SessionRegistry.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/registry/SessionRegistry.java @@ -16,8 +16,15 @@ */ package com.alipay.sofa.registry.server.session.registry; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; -import com.alipay.sofa.registry.common.model.store.BaseInfo; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.StoreData; import com.alipay.sofa.registry.common.model.store.Subscriber; @@ -28,114 +35,171 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.Server; import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.acceptor.WriteDataAcceptor; +import com.alipay.sofa.registry.server.session.acceptor.WriteDataRequest; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.filter.DataIdMatchStrategy; import com.alipay.sofa.registry.server.session.node.NodeManager; import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.renew.RenewService; import com.alipay.sofa.registry.server.session.store.DataStore; import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.store.Watchers; import com.alipay.sofa.registry.server.session.strategy.SessionRegistryStrategy; +import com.alipay.sofa.registry.server.session.wrapper.Wrapper; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInterceptorManager; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInvocation; import com.alipay.sofa.registry.task.listener.TaskEvent; -import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; /** - * * @author shangyu.wh * @version $Id: AbstractSessionRegistry.java, v 0.1 2017-11-30 18:13 shangyu.wh Exp $ */ public class SessionRegistry implements Registry { - private static final Logger LOGGER = LoggerFactory.getLogger(SessionRegistry.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(SessionRegistry.class); + + private static final Logger TASK_LOGGER = LoggerFactory.getLogger( + SessionRegistry.class, "[Task]"); - private static final Logger TASK_LOGGER = LoggerFactory.getLogger(SessionRegistry.class, - "[Task]"); + private static final Logger RENEW_LOGGER = LoggerFactory.getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[SessionRegistry]"); /** * store subscribers */ @Autowired - private Interests sessionInterests; + private Interests sessionInterests; /** * store watchers */ @Autowired - private Watchers sessionWatchers; + private Watchers sessionWatchers; /** * store publishers */ @Autowired - private DataStore sessionDataStore; + private DataStore sessionDataStore; /** * transfer data to DataNode */ @Autowired - private DataNodeService dataNodeService; + private DataNodeService dataNodeService; /** * trigger task com.alipay.sofa.registry.server.meta.listener process */ @Autowired - private TaskListenerManager taskListenerManager; + private TaskListenerManager taskListenerManager; /** * calculate data node url */ @Autowired - private NodeManager dataNodeManager; + private NodeManager dataNodeManager; @Autowired - private SessionServerConfig sessionServerConfig; + private SessionServerConfig sessionServerConfig; @Autowired - private Exchange boltExchange; + private Exchange boltExchange; @Autowired - private SessionRegistryStrategy sessionRegistryStrategy; - - @Override - public void register(StoreData storeData) { - - //check connect already existed - checkConnect(storeData); - - switch (storeData.getDataType()) { - case PUBLISHER: - Publisher publisher = (Publisher) storeData; + private SessionRegistryStrategy sessionRegistryStrategy; - dataNodeService.register(publisher); + @Autowired + private WrapperInterceptorManager wrapperInterceptorManager; - sessionDataStore.add(publisher); + @Autowired + private DataIdMatchStrategy dataIdMatchStrategy; - sessionRegistryStrategy.afterPublisherRegister(publisher); - break; - case SUBSCRIBER: - Subscriber subscriber = (Subscriber) storeData; + @Autowired + private RenewService renewService; - sessionInterests.add(subscriber); + @Autowired + private WriteDataAcceptor writeDataAcceptor; - sessionRegistryStrategy.afterSubscriberRegister(subscriber); - break; - case WATCHER: - Watcher watcher = (Watcher) storeData; + private volatile boolean enableDataRenewSnapshot = true; - sessionWatchers.add(watcher); + @Override + public void register(StoreData storeData) { - sessionRegistryStrategy.afterWatcherRegister(watcher); - break; - default: - break; + WrapperInvocation wrapperInvocation = new WrapperInvocation( + new Wrapper() { + @Override + public Boolean call() { + + switch (storeData.getDataType()) { + case PUBLISHER: + Publisher publisher = (Publisher) storeData; + + sessionDataStore.add(publisher); + + // All write operations to DataServer (pub/unPub/clientoff/renew/snapshot) + // are handed over to WriteDataAcceptor + writeDataAcceptor.accept(new WriteDataRequest() { + @Override + public Object getRequestBody() { + return publisher; + } + + @Override + public WriteDataRequestType getRequestType() { + return WriteDataRequestType.PUBLISHER; + } + + @Override + public String getConnectId() { + return publisher.getSourceAddress().getAddressString() + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString(); + } + + @Override + public String getDataServerIP() { + Node dataNode = dataNodeManager.getNode(publisher.getDataInfoId()); + return dataNode.getNodeUrl().getIpAddress(); + } + }); + + sessionRegistryStrategy.afterPublisherRegister(publisher); + break; + case SUBSCRIBER: + Subscriber subscriber = (Subscriber) storeData; + + sessionInterests.add(subscriber); + + sessionRegistryStrategy.afterSubscriberRegister(subscriber); + break; + case WATCHER: + Watcher watcher = (Watcher) storeData; + + sessionWatchers.add(watcher); + + sessionRegistryStrategy.afterWatcherRegister(watcher); + break; + default: + break; + } + return null; + } + + @Override + public Supplier getParameterSupplier() { + return () -> storeData; + } + + }, wrapperInterceptorManager); + + try { + wrapperInvocation.proceed(); + } catch (Exception e) { + throw new RuntimeException("Proceed register error!", e); } } @@ -149,7 +213,32 @@ public void unRegister(StoreData storeData) { sessionDataStore.deleteById(storeData.getId(), publisher.getDataInfoId()); - dataNodeService.unregister(publisher); + // All write operations to DataServer (pub/unPub/clientoff/renew/snapshot) + // are handed over to WriteDataAcceptor + writeDataAcceptor.accept(new WriteDataRequest() { + @Override + public Object getRequestBody() { + return publisher; + } + + @Override + public WriteDataRequestType getRequestType() { + return WriteDataRequestType.UN_PUBLISHER; + } + + @Override + public String getConnectId() { + return publisher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString(); + } + + @Override + public String getDataServerIP() { + Node dataNode = dataNodeManager.getNode(publisher.getDataInfoId()); + return dataNode.getNodeUrl().getIpAddress(); + } + }); sessionRegistryStrategy.afterPublisherUnRegister(publisher); break; @@ -175,9 +264,52 @@ public void unRegister(StoreData storeData) { @Override public void cancel(List connectIds) { - TaskEvent taskEvent = new TaskEvent(connectIds, TaskType.CANCEL_DATA_TASK); - TASK_LOGGER.info("send " + taskEvent.getTaskType() + " taskEvent:{}", taskEvent); - taskListenerManager.sendTaskEvent(taskEvent); + //update local firstly, data node send error depend on renew check + List connectIdsWithPub = new ArrayList<>(); + removeFromSession(connectIds, connectIdsWithPub); + + // clientOff to dataNode async + clientOffToDataNode(connectIdsWithPub); + + } + + private void removeFromSession(List connectIds, List connectIdsWithPub) { + for (String connectId : connectIds) { + if (sessionDataStore.deleteByConnectId(connectId)) { + connectIdsWithPub.add(connectId); + } + sessionInterests.deleteByConnectId(connectId); + sessionWatchers.deleteByConnectId(connectId); + } + } + + private void clientOffToDataNode(List connectIdsWithPub) { + // All write operations to DataServer (pub/unPub/clientoff/renew/snapshot) + // are handed over to WriteDataAcceptor + for (String connectId : connectIdsWithPub) { + writeDataAcceptor.accept(new WriteDataRequest() { + @Override + public Object getRequestBody() { + return connectId; + } + + @Override + public WriteDataRequestType getRequestType() { + return WriteDataRequestType.CLIENT_OFF; + } + + @Override + public String getConnectId() { + return connectId; + } + + @Override + public String getDataServerIP() { + return null; + } + }); + writeDataAcceptor.remove(connectId); + } } @Override @@ -196,7 +328,6 @@ public void fetchChangData() { @Override public void fetchChangDataProcess() { - //check dataInfoId's sub list is not empty List checkDataInfoIds = new ArrayList<>(); sessionInterests.getInterestDataInfoIds().forEach((dataInfoId) -> { @@ -205,6 +336,9 @@ public void fetchChangDataProcess() { checkDataInfoIds.add(dataInfoId); } }); + + LOGGER.info("[fetchChangDataProcess] Fetch data versions for {} dataInfoIds", checkDataInfoIds.size()); + Map/*dataInfoIds*/> map = calculateDataNode(checkDataInfoIds); map.forEach((address, dataInfoIds) -> { @@ -228,10 +362,8 @@ private Map> calculateDataNode(Collection dat if (dataInfoIds != null) { dataInfoIds.forEach(dataInfoId -> { Node dataNode = dataNodeManager.getNode(dataInfoId); - URL url = new URL(dataNode.getNodeUrl().getIpAddress(), - sessionServerConfig.getDataServerPort()); - Collection list = map.computeIfAbsent(url.getAddressString(), - k -> new ArrayList<>()); + URL url = new URL(dataNode.getNodeUrl().getIpAddress(), sessionServerConfig.getDataServerPort()); + Collection list = map.computeIfAbsent(url.getAddressString(), k -> new ArrayList<>()); list.add(dataInfoId); }); } @@ -239,20 +371,142 @@ private Map> calculateDataNode(Collection dat return map; } - private void checkConnect(StoreData storeData) { + public void remove(List connectIds) { + + List connectIdsAll = new ArrayList<>(); + connectIds.forEach(connectId -> { + Map pubMap = getSessionDataStore().queryByConnectId(connectId); + boolean pubExisted = pubMap != null && !pubMap.isEmpty(); + + Map subMap = getSessionInterests().queryByConnectId(connectId); + boolean subExisted = false; + if (subMap != null && !subMap.isEmpty()) { + subExisted = true; + + subMap.forEach((registerId, sub) -> { + if (dataIdMatchStrategy + .match(sub.getDataId(), () -> sessionServerConfig.getBlacklistSubDataIdRegex())) { + fireSubscriberPushEmptyTask(sub); + } + }); + } + + if (pubExisted || subExisted) { + connectIdsAll.add(connectId); + } + }); + if (!connectIds.isEmpty()) { + TaskEvent taskEvent = new TaskEvent(connectIds, TaskEvent.TaskType.CANCEL_DATA_TASK); + TASK_LOGGER.info("send " + taskEvent.getTaskType() + " taskEvent:{}", taskEvent); + getTaskListenerManager().sendTaskEvent(taskEvent); + } + } + + private void fireSubscriberPushEmptyTask(Subscriber subscriber) { + //trigger empty data push + TaskEvent taskEvent = new TaskEvent(subscriber, + TaskEvent.TaskType.SUBSCRIBER_PUSH_EMPTY_TASK); + TASK_LOGGER.info("send " + taskEvent.getTaskType() + " taskEvent:{}", taskEvent); + getTaskListenerManager().sendTaskEvent(taskEvent); + } + + public void cleanClientConnect() { - BaseInfo baseInfo = (BaseInfo) storeData; + Set connectIndexes = new HashSet<>(); + Set pubIndexes = sessionDataStore.getConnectPublishers().keySet(); + Set subIndexes = sessionInterests.getConnectSubscribers().keySet(); + Set watchIndexes = sessionWatchers.getConnectWatchers().keySet(); + connectIndexes.addAll(pubIndexes); + connectIndexes.addAll(subIndexes); + connectIndexes.addAll(watchIndexes); Server sessionServer = boltExchange.getServer(sessionServerConfig.getServerPort()); - Channel channel = sessionServer.getChannel(baseInfo.getSourceAddress()); + List connectIds = new ArrayList<>(); + for (String connectId : connectIndexes) { + String[] parts = connectId.split(ValueConstants.CONNECT_ID_SPLIT); + Channel channel = sessionServer.getChannel(URL.valueOf(parts[0])); + if (channel == null) { + connectIds.add(connectId); + LOGGER.warn("Client connect has not existed!it must be remove!connectId:{}", + connectId); + } + } + if (!connectIds.isEmpty()) { + cancel(connectIds); + } + } - if (channel == null) { - throw new RuntimeException(String.format( - "Register address %s has not connected session server!", - baseInfo.getSourceAddress())); + @Override + public void renewDatum(String connectId) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("renewDatum: connectId={}", connectId); + } + // check the renew switch + if (!this.enableDataRenewSnapshot) { + return; } + List renewDatumRequests = renewService.getRenewDatumRequests(connectId); + if (renewDatumRequests != null) { + for (RenewDatumRequest renewDatumRequest : renewDatumRequests) { + // All write operations to DataServer (pub/unPub/clientoff/renew/snapshot) + // are handed over to WriteDataAcceptor + writeDataAcceptor.accept(new WriteDataRequest() { + @Override + public Object getRequestBody() { + return renewDatumRequest; + } + + @Override + public WriteDataRequestType getRequestType() { + return WriteDataRequestType.RENEW_DATUM; + } + + @Override + public String getConnectId() { + return connectId; + } + + @Override + public String getDataServerIP() { + return renewDatumRequest.getDataServerIP(); + } + }); + } + } + } + + @Override + public void sendDatumSnapshot(String connectId, String dataServerIP) { + if (RENEW_LOGGER.isDebugEnabled()) { + RENEW_LOGGER.debug("sendDatumSnapshot: connectId={}, dataServerIP={}", connectId, + dataServerIP); + } + + // All write operations to DataServer (pub/unPub/clientoff/renew/snapshot) + // are handed over to WriteDataAcceptor + writeDataAcceptor.accept(new WriteDataRequest() { + @Override + public Object getRequestBody() { + return connectId; + } + + @Override + public WriteDataRequestType getRequestType() { + return WriteDataRequestType.DATUM_SNAPSHOT; + } + + @Override + public String getConnectId() { + return connectId; + } + + @Override + public String getDataServerIP() { + return dataServerIP; + } + }); } /** @@ -260,7 +514,7 @@ private void checkConnect(StoreData storeData) { * * @return property value of sessionInterests */ - public Interests getSessionInterests() { + protected Interests getSessionInterests() { return sessionInterests; } @@ -269,7 +523,7 @@ public Interests getSessionInterests() { * * @return property value of sessionDataStore */ - public DataStore getSessionDataStore() { + protected DataStore getSessionDataStore() { return sessionDataStore; } @@ -278,7 +532,17 @@ public DataStore getSessionDataStore() { * * @return property value of taskListenerManager */ - public TaskListenerManager getTaskListenerManager() { + protected TaskListenerManager getTaskListenerManager() { return taskListenerManager; } + + /** + * Setter method for property enableDataRenewSnapshot . + * + * @param enableDataRenewSnapshot value to be assigned to property enableDataRenewSnapshot + */ + @Override + public void setEnableDataRenewSnapshot(boolean enableDataRenewSnapshot) { + this.enableDataRenewSnapshot = enableDataRenewSnapshot; + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/ClientNodeExchanger.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/ClientNodeExchanger.java index 86d5bedba..686534cac 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/ClientNodeExchanger.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/ClientNodeExchanger.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.remoting; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -29,7 +31,6 @@ import com.alipay.sofa.registry.remoting.exchange.message.Response; import com.alipay.sofa.registry.remoting.exchange.message.Response.ResultStatus; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -62,12 +63,12 @@ public Response request(Request request) throws RequestException { //TODO log ASYNC sessionServer.sendCallback(channel, request.getRequestBody(), request.getCallBackHandler(), - sessionServerConfig.getClientNodeExchangeTimeOut()); + request.getTimeout() != null ? request.getTimeout() : sessionServerConfig.getClientNodeExchangeTimeOut()); response = () -> ResultStatus.SUCCESSFUL; } else { final Object result = sessionServer.sendSync(channel, request.getRequestBody(), - sessionServerConfig.getClientNodeExchangeTimeOut()); + request.getTimeout() != null ? request.getTimeout() : sessionServerConfig.getClientNodeExchangeTimeOut()); response = () -> result; } } else { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/DataNodeExchanger.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/DataNodeExchanger.java index fb49eb26f..da2ecc9cf 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/DataNodeExchanger.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/DataNodeExchanger.java @@ -16,11 +16,18 @@ */ package com.alipay.sofa.registry.server.session.remoting; +import java.util.Collection; +import java.util.StringJoiner; + +import javax.annotation.Resource; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.Client; import com.alipay.sofa.registry.remoting.exchange.Exchange; @@ -31,10 +38,6 @@ import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.NodeManager; import com.alipay.sofa.registry.server.session.remoting.handler.AbstractClientHandler; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Resource; -import java.util.Collection; /** * The type Data node exchanger. @@ -69,42 +72,30 @@ public Response request(Request request) throws RequestException { Response response; URL url = request.getRequestUrl(); try { - Client sessionClient = boltExchange.getClient(Exchange.DATA_SERVER_TYPE); - - if (sessionClient == null) { - LOGGER.warn( - "DataNode Exchanger get dataServer connection {} error! Connection can not be null or disconnected!", - url); - //first start session maybe case sessionClient null,try to auto connect - sessionClient = boltExchange.connect(Exchange.DATA_SERVER_TYPE, url, - dataClientHandlers.toArray(new ChannelHandler[dataClientHandlers.size()])); - } - //try to connect data - Channel channel = sessionClient.getChannel(url); - if (channel == null) { - channel = sessionClient.connect(url); + Client sessionClient = getClient(url); + + // print but ignore if from renew module, cause renew request is too much + if (!(request.getRequestBody() instanceof RenewDatumRequest)) { + EXCHANGE_LOGGER.info("DataNode Exchanger request={},url={}", request.getRequestBody(), url); } - EXCHANGE_LOGGER.info("DataNode Exchanger request={},url={}", request.getRequestBody(), url); - final Object result = sessionClient.sendSync(channel, request.getRequestBody(), - sessionServerConfig.getDataNodeExchangeTimeOut()); + final Object result = sessionClient + .sendSync(url, request.getRequestBody(), request.getTimeout() != null ? request.getTimeout() : sessionServerConfig.getDataNodeExchangeTimeOut()); if (result == null) { - LOGGER.error("DataNode Exchanger request data get null result!Request url:" + url); - throw new RequestException("DataNode Exchanger request data get null result!", - request); + throw new RequestException("DataNode Exchanger request data get null result!", request); } response = () -> result; } catch (Exception e) { - LOGGER.error("DataNode Exchanger request data error!request={},url={}", request.getRequestBody(), url, e); - throw new RequestException("DataNode Exchanger request data error!Request url:" + url, - request, e); + LOGGER.error(String.format("Error when request DataNode! Request url=%s, request=%s, msg=%s", url, + request.getRequestBody(), e.getMessage())); + throw new RequestException("DataNode Exchanger request data error! Request url:" + url, request, e); } return response; } @Override - public Client connectServer() { + public synchronized Client connectServer() { Collection dataNodes = dataNodeManager.getDataCenterNodes(); if (dataNodes == null || dataNodes.isEmpty()) { @@ -112,7 +103,11 @@ public Client connectServer() { dataNodes = dataNodeManager.getDataCenterNodes(); } + boolean connectedAll = true; + StringJoiner errorMsg = new StringJoiner(";"); + Client dataClient = null; + for (Node dataNode : dataNodes) { if (dataNode.getNodeUrl() == null || dataNode.getNodeUrl().getIpAddress() == null) { LOGGER.error("get data node address error!url{}", dataNode.getNodeUrl()); @@ -122,25 +117,33 @@ public Client connectServer() { URL url = new URL(dataNode.getNodeUrl().getIpAddress(), sessionServerConfig.getDataServerPort()); try { - dataClient = boltExchange.getClient(Exchange.DATA_SERVER_TYPE); - if (dataClient == null) { - dataClient = boltExchange.connect(Exchange.DATA_SERVER_TYPE, url, - dataClientHandlers.toArray(new ChannelHandler[dataClientHandlers.size()])); - } + dataClient = getClient(url); + // make sure there are connections to DataServer + dataClient.connect(url); } catch (Exception e) { - LOGGER.error("DataNode Exchanger connect DataServer error!url:" + url, e); + String msg = "DataNode Exchanger connect DataServer error!url:" + url; + LOGGER.error(msg, e); + connectedAll = false; + errorMsg.add(msg); continue; } + } - try { - Channel channel = dataClient.getChannel(url); - if (channel == null) { - dataClient.connect(url); - } - } catch (Exception e) { - LOGGER.error("DataNode Exchanger connect channel error!url:" + url, e); - } + if (!connectedAll) { + throw new RuntimeException("Data server connected server error: " + errorMsg.toString()); } return dataClient; } + + protected Client getClient(URL url) { + Client sessionClient = boltExchange.getClient(Exchange.DATA_SERVER_TYPE); + if (sessionClient == null) { + //first start session maybe case sessionClient null,try to auto connect + sessionClient = boltExchange.connect(Exchange.DATA_SERVER_TYPE, + sessionServerConfig.getDataClientConnNum(), url, + dataClientHandlers.toArray(new ChannelHandler[dataClientHandlers.size()])); + } + return sessionClient; + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/MetaNodeExchanger.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/MetaNodeExchanger.java index 5f66bf863..4bf823c0b 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/MetaNodeExchanger.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/MetaNodeExchanger.java @@ -16,6 +16,12 @@ */ package com.alipay.sofa.registry.server.session.remoting; +import java.util.Collection; + +import javax.annotation.Resource; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.metaserver.MetaNode; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; @@ -32,10 +38,6 @@ import com.alipay.sofa.registry.server.session.node.NodeManager; import com.alipay.sofa.registry.server.session.node.RaftClientManager; import com.alipay.sofa.registry.server.session.remoting.handler.AbstractClientHandler; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.annotation.Resource; -import java.util.Collection; /** * The type Data node exchanger. @@ -80,29 +82,20 @@ public Response request(Request request) throws RequestException { sessionClient = boltExchange.connect(Exchange.META_SERVER_TYPE, url, metaClientHandlers.toArray(new ChannelHandler[metaClientHandlers.size()])); } - //try to connect data - Channel channel = sessionClient.getChannel(url); - if (channel == null) { - channel = sessionClient.connect(url); - } try { - final Object result = sessionClient.sendSync(channel, request.getRequestBody(), - sessionServerConfig.getDataNodeExchangeTimeOut()); + final Object result = sessionClient.sendSync(url, request.getRequestBody(), + request.getTimeout() != null ? request.getTimeout() : sessionServerConfig.getDataNodeExchangeTimeOut()); response = () -> result; } catch (Exception e) { //retry url = new URL(raftClientManager.refreshLeader().getIp(), sessionServerConfig.getMetaServerPort()); - channel = sessionClient.getChannel(url); - if (channel == null) { - channel = sessionClient.connect(url); - } LOGGER.warn("MetaNode Exchanger request send error!It will be retry once!Request url:{}", url); - final Object result = sessionClient.sendSync(channel, request.getRequestBody(), - sessionServerConfig.getDataNodeExchangeTimeOut()); + final Object result = sessionClient.sendSync(url, request.getRequestBody(), + request.getTimeout() != null ? request.getTimeout() : sessionServerConfig.getDataNodeExchangeTimeOut()); response = () -> result; } } catch (Exception e) { @@ -115,7 +108,7 @@ public Response request(Request request) throws RequestException { } @Override - public Client connectServer() { + public synchronized Client connectServer() { Collection nodes = metaNodeManager.getDataCenterNodes(); if (nodes == null || nodes.isEmpty()) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractClientHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractClientHandler.java index 7cbeebdc3..b9f0c3a90 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractClientHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractClientHandler.java @@ -22,6 +22,10 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.RemotingException; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.Executor; /** * @@ -30,6 +34,9 @@ */ public abstract class AbstractClientHandler implements ChannelHandler { + @Autowired + private ExecutorManager executorManager; + private static final Logger LOGGER = LoggerFactory.getLogger("SESSION-CONNECT"); @Override @@ -71,4 +78,9 @@ public Object reply(Channel channel, T message) { public Class interest() { return null; } + + @Override + public Executor getExecutor() { + return executorManager.getDefaultRequestExecutor(); + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractServerHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractServerHandler.java index bf8e42e81..d0a6024bb 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractServerHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/AbstractServerHandler.java @@ -22,6 +22,10 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; import com.alipay.sofa.registry.remoting.RemotingException; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.Executor; /** * @@ -30,6 +34,9 @@ */ public abstract class AbstractServerHandler implements ChannelHandler { + @Autowired + private ExecutorManager executorManager; + private static final Logger LOGGER = LoggerFactory.getLogger("SESSION-CONNECT"); @Override @@ -73,4 +80,9 @@ public Object reply(Channel channel, T message) throws RemotingException { public Class interest() { return null; } + + @Override + public Executor getExecutor() { + return executorManager.getDefaultRequestExecutor(); + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ClientNodeConnectionHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ClientNodeConnectionHandler.java index 6c0dab457..d515621af 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ClientNodeConnectionHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ClientNodeConnectionHandler.java @@ -16,21 +16,34 @@ */ package com.alipay.sofa.registry.server.session.remoting.handler; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + +import org.apache.commons.lang.math.RandomUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.RemotingException; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.registry.Registry; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.store.DataStore; import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.store.Watchers; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; +import com.alipay.sofa.registry.timer.RecycleAsyncHashedWheelTimer; +import com.google.common.util.concurrent.ThreadFactoryBuilder; /** * @@ -39,40 +52,81 @@ */ public class ClientNodeConnectionHandler extends AbstractServerHandler { - private static final Logger LOGGER = LoggerFactory.getLogger("SESSION-CONNECT"); + private static final Logger LOGGER = LoggerFactory.getLogger("SESSION-CONNECT"); + private static final Logger RENEW_LOGGER = LoggerFactory.getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[ClientNodeConnectionHandler]"); + + private RecycleAsyncHashedWheelTimer recycleAsyncHashedWheelTimer; @Autowired - private Registry sessionRegistry; + private Registry sessionRegistry; @Autowired - private DataStore sessionDataStore; + private DataStore sessionDataStore; @Autowired - private Interests sessionInterests; + private Interests sessionInterests; @Autowired - private Watchers sessionWatchers; + private Watchers sessionWatchers; @Autowired - private ExecutorManager executorManager; + private ExecutorManager executorManager; + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + private Exchange boltExchange; + + @PostConstruct + public void init() { + ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); + threadFactoryBuilder.setDaemon(true); + recycleAsyncHashedWheelTimer = new RecycleAsyncHashedWheelTimer(threadFactoryBuilder + .setNameFormat("Registry-RenewDatumTask-WheelTimer").build(), + sessionServerConfig.getRenewDatumWheelTicksDuration(), TimeUnit.MILLISECONDS, + sessionServerConfig.getRenewDatumWheelTicksSize(), + sessionServerConfig.getRenewDatumWheelThreadSize(), + sessionServerConfig.getRenewDatumWheelQueueSize(), threadFactoryBuilder.setNameFormat( + "Registry-RenewDatumTask-WheelExecutor-%d").build(), new TaskFailedCallback() { + @Override + public void executionRejected(Throwable e) { + RENEW_LOGGER.error("executionRejected: " + e.getMessage(), e); + } + + @Override + public void executionFailed(Throwable e) { + RENEW_LOGGER.error("executionFailed: " + e.getMessage(), e); + } + }); + } @Override public HandlerType getType() { return HandlerType.LISENTER; } + @Override + public void connected(Channel channel) throws RemotingException { + super.connected(channel); + fireRenewDatum(channel); + } + @Override public void disconnected(Channel channel) throws RemotingException { super.disconnected(channel); fireCancelClient(channel); } - public void fireCancelClient(Channel channel) { + private void fireCancelClient(Channel channel) { //avoid block connect ConnectionEventExecutor thread pool - executorManager.getDisconnectClientExecutor().execute(()->{ + executorManager.getConnectClientExecutor().execute(() -> { - String connectId = NetUtil.toAddressString(channel.getRemoteAddress()); - if(checkCache(connectId)) { + String connectId = NetUtil.toAddressString(channel.getRemoteAddress()) + ValueConstants.CONNECT_ID_SPLIT + + NetUtil.toAddressString(channel.getLocalAddress()); + if (checkCache(connectId)) { List connectIds = new ArrayList<>(); connectIds.add(connectId); sessionRegistry.cancel(connectIds); @@ -103,4 +157,30 @@ private boolean checkWatcher(String connectId) { Map subMap = sessionWatchers.queryByConnectId(connectId); return subMap != null && !subMap.isEmpty(); } + + private void fireRenewDatum(Channel channel) { + executorManager.getConnectClientExecutor().execute(() -> { + String connectId = NetUtil.toAddressString(channel.getRemoteAddress()) + ValueConstants.CONNECT_ID_SPLIT + + NetUtil.toAddressString(channel.getLocalAddress()); + RENEW_LOGGER.info("Renew task is started: {}", connectId); + recycleAsyncHashedWheelTimer.newTimeout(timerOut -> sessionRegistry.renewDatum(connectId), randomDelay(), + sessionServerConfig.getRenewDatumWheelTaskDelaySec(), TimeUnit.SECONDS, () -> { + Server sessionServer = boltExchange.getServer(sessionServerConfig.getServerPort()); + Channel channelClient = sessionServer.getChannel(URL.valueOf(connectId.split(ValueConstants.CONNECT_ID_SPLIT)[0])); + boolean shouldContinue = channelClient != null && channel.isConnected(); + if (!shouldContinue) { + RENEW_LOGGER.info("Renew task is stop: {}", connectId); + } + return shouldContinue; + }); + }); + } + + private long randomDelay() { + return sessionServerConfig.getRenewDatumWheelTaskRandomFirstDelaySec() + / 2 + + RandomUtils.nextInt(sessionServerConfig + .getRenewDatumWheelTaskRandomFirstDelaySec() / 2); + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ConfigureLoadbalanceHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ConfigureLoadbalanceHandler.java new file mode 100644 index 000000000..7f0c075c7 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/ConfigureLoadbalanceHandler.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.remoting.handler; + +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.metaserver.ConfigureLoadbalanceRequest; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.session.connections.ConnectionsService; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @author xiangxu + * @version : ConfigureLoadbalanceHandler.java, v 0.1 2020年05月29日 11:51 上午 xiangxu Exp $ + */ +public class ConfigureLoadbalanceHandler extends AbstractClientHandler { + + @Autowired + private ConnectionsService connectionsService; + + @Override + protected Node.NodeType getConnectNodeType() { + return Node.NodeType.SESSION; + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Class interest() { + return ConfigureLoadbalanceRequest.class; + } + + @Override + public Object reply(Channel channel, Object message) { + + ConfigureLoadbalanceRequest configureLoadbalanceRequest = (ConfigureLoadbalanceRequest) message; + + connectionsService.setMaxConnections(configureLoadbalanceRequest.getMaxConnections()); + return null; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataChangeRequestHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataChangeRequestHandler.java index bc70a50c9..04cd02705 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataChangeRequestHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataChangeRequestHandler.java @@ -16,6 +16,10 @@ */ package com.alipay.sofa.registry.server.session.remoting.handler; +import java.util.concurrent.Executor; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.sessionserver.DataChangeRequest; import com.alipay.sofa.registry.log.Logger; @@ -29,12 +33,10 @@ import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.strategy.DataChangeRequestHandlerStrategy; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.concurrent.Executor; /** * + * @author kezhu.wukz * @author shangyu.wh * @version $Id: DataChangeRequestHandler.java, v 0.1 2017-12-12 15:09 shangyu.wh Exp $ */ @@ -81,7 +83,6 @@ public Executor getExecutor() { @Override public Object reply(Channel channel, Object message) { - DataChangeRequest dataChangeRequest = (DataChangeRequest) message; dataChangeRequest.setDataCenter(dataChangeRequest.getDataCenter()); @@ -92,13 +93,14 @@ public Object reply(Channel channel, Object message) { dataChangeRequest.getDataInfoId(), dataChangeRequest.getDataCenter()))); if (sessionServerConfig.isStopPushSwitch()) { - LOGGER.info("Stop Push data with switch on,dataChangeRequest: {}", dataChangeRequest); return null; } + try { boolean result = sessionInterests.checkInterestVersions( dataChangeRequest.getDataCenter(), dataChangeRequest.getDataInfoId(), dataChangeRequest.getVersion()); + if (!result) { return null; } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataNodeConnectionHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataNodeConnectionHandler.java index e5f29820a..3e1ed841c 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataNodeConnectionHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/DataNodeConnectionHandler.java @@ -16,8 +16,9 @@ */ package com.alipay.sofa.registry.server.session.remoting.handler; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node.NodeType; -import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; @@ -26,7 +27,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -52,7 +52,7 @@ public HandlerType getType() { @Override public void connected(Channel channel) throws RemotingException { super.connected(channel); - fireRegisterProcessIdTask(new URL(channel.getRemoteAddress())); + fireRegisterProcessIdTask(channel); } @Override @@ -60,8 +60,8 @@ protected NodeType getConnectNodeType() { return NodeType.DATA; } - private void fireRegisterProcessIdTask(URL dataURL) { - TaskEvent taskEvent = new TaskEvent(dataURL, TaskType.SESSION_REGISTER_DATA_TASK); + private void fireRegisterProcessIdTask(Channel boltChannel) { + TaskEvent taskEvent = new TaskEvent(boltChannel, TaskType.SESSION_REGISTER_DATA_TASK); taskLogger.info("send " + taskEvent.getTaskType() + " taskEvent:{}", taskEvent); taskListenerManager.sendTaskEvent(taskEvent); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/LoadbalanceMetricsHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/LoadbalanceMetricsHandler.java new file mode 100644 index 000000000..075adf29c --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/LoadbalanceMetricsHandler.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.remoting.handler; + +import com.alipay.sofa.registry.common.model.Node.NodeType; +import com.alipay.sofa.registry.common.model.metaserver.GetLoadbalanceMetricsRequest; +import com.alipay.sofa.registry.common.model.metaserver.LoadbalanceMetrics; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.session.connections.ConnectionsService; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @author xiangxu + * @version : LoadbalanceMetricsHandler.java, v 0.1 2020年05月27日 2:56 下午 xiangxu Exp $ + */ +public class LoadbalanceMetricsHandler extends AbstractClientHandler { + + @Autowired + private ConnectionsService connectionsService; + + @Override + protected NodeType getConnectNodeType() { + return NodeType.SESSION; + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Object reply(Channel channel, Object message) { + LoadbalanceMetrics m = new LoadbalanceMetrics(); + m.setConnectionCount(connectionsService.getConnections().size()); + return m; + } + + @Override + public Class interest() { + return GetLoadbalanceMetricsRequest.class; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/NotifyProvideDataChangeHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/NotifyProvideDataChangeHandler.java index eb59882c1..57ca25d32 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/NotifyProvideDataChangeHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/NotifyProvideDataChangeHandler.java @@ -68,7 +68,10 @@ public Object reply(Channel channel, Object message) { NotifyProvideDataChange notifyProvideDataChange = (NotifyProvideDataChange) message; if (!ValueConstants.STOP_PUSH_DATA_SWITCH_DATA_ID.equals(notifyProvideDataChange - .getDataInfoId())) { + .getDataInfoId()) + && !ValueConstants.BLACK_LIST_DATA_ID.equals(notifyProvideDataChange.getDataInfoId()) + && !ValueConstants.ENABLE_DATA_RENEW_SNAPSHOT.equals(notifyProvideDataChange + .getDataInfoId())) { boolean result = sessionWatchers.checkWatcherVersions( notifyProvideDataChange.getDataInfoId(), notifyProvideDataChange.getVersion()); if (!result) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/WatcherHandler.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/WatcherHandler.java index 07fd647ab..aa8825973 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/WatcherHandler.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/remoting/handler/WatcherHandler.java @@ -16,11 +16,15 @@ */ package com.alipay.sofa.registry.server.session.remoting.handler; +import java.util.concurrent.Executor; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.core.model.ConfiguratorRegister; import com.alipay.sofa.registry.core.model.RegisterResponse; import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.strategy.WatcherHandlerStrategy; -import org.springframework.beans.factory.annotation.Autowired; /** * @@ -28,6 +32,9 @@ * @version $Id: SubscriberHandler.java, v 0.1 2017-11-30 15:01 shangyu.wh Exp $ */ public class WatcherHandler extends AbstractServerHandler { + @Autowired + private ExecutorManager executorManager; + @Autowired private WatcherHandlerStrategy watcherHandlerStrategy; @@ -49,4 +56,9 @@ public Class interest() { return ConfiguratorRegister.class; } + @Override + public Executor getExecutor() { + return executorManager.getAccessDataExecutor(); + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/DefaultRenewService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/DefaultRenewService.java new file mode 100644 index 000000000..c630212e9 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/DefaultRenewService.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.renew; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.PublisherDigestUtil; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.session.node.NodeManager; +import com.alipay.sofa.registry.server.session.store.DataStore; + +/** + * + * @author kezhu.wukz + * @version $Id: DefaultRenewService.java, v 0.1 2019-06-27 11:09 kezhu.wukz Exp $ + */ +public class DefaultRenewService implements RenewService { + + /*** store publishers */ + @Autowired + private DataStore sessionDataStore; + + /*** calculate data node url */ + @Autowired + private NodeManager dataNodeManager; + + @Override + public List getRenewDatumRequests(String connectId) { + List datumSnapshotRequests = getDatumSnapshotRequest(connectId); + if (datumSnapshotRequests != null && !datumSnapshotRequests.isEmpty()) { + return datumSnapshotRequests.stream() + .map(datumSnapshotRequest -> new RenewDatumRequest(datumSnapshotRequest.getConnectId(), + datumSnapshotRequest.getDataServerIp(), String.valueOf( + PublisherDigestUtil.getDigestValueSum(datumSnapshotRequest.getPublishers())))) + .collect(Collectors.toList()); + } + return null; + } + + @Override + public List getDatumSnapshotRequest(String connectId) { + Map pubMap = sessionDataStore.queryByConnectId(connectId); + if (pubMap != null && !pubMap.isEmpty()) { + Map> dataServerIpToPubs = new ConcurrentHashMap<>(); + List list = new ArrayList<>(); + pubMap.values().forEach(publisher -> { + Node dataNode = dataNodeManager.getNode(publisher.getDataInfoId()); + List publishers = dataServerIpToPubs + .computeIfAbsent(dataNode.getNodeUrl().getIpAddress(), k -> new ArrayList<>()); + publishers.add(publisher); + }); + for (Map.Entry> entry : dataServerIpToPubs.entrySet()) { + List publishers = entry.getValue(); + if (!publishers.isEmpty()) { + DatumSnapshotRequest datumSnapshotRequest = new DatumSnapshotRequest(connectId, entry.getKey(), + publishers); + list.add(datumSnapshotRequest); + } + } + return list; + } + return null; + } + + @Override + public DatumSnapshotRequest getDatumSnapshotRequest(String connectId, String dataServerIP) { + List publishers = new ArrayList<>(); + Map pubMap = sessionDataStore.queryByConnectId(connectId); + if (pubMap != null && !pubMap.isEmpty()) { + pubMap.values().forEach(publisher -> { + Node dataNode = dataNodeManager.getNode(publisher.getDataInfoId()); + if (dataServerIP.equalsIgnoreCase(dataNode.getNodeUrl().getIpAddress())) { + publishers.add(publisher); + } + }); + } + if (!publishers.isEmpty()) { + return new DatumSnapshotRequest(connectId, dataServerIP, publishers); + } + return null; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/RenewService.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/RenewService.java new file mode 100644 index 000000000..ad760951f --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/renew/RenewService.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.renew; + +import java.util.List; + +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.common.model.RenewDatumRequest; + +/** + * + * @author kezhu.wukz + * @version $Id: RenewService.java, v 0.1 2019-06-27 11:06 kezhu.wukz Exp $ + */ +public interface RenewService { + + /** + * Get RenewDatumRequests based on connectId + * + * @param connectId + * @return + */ + List getRenewDatumRequests(String connectId); + + /** + * Get DatumSnapshotRequests based on connectId + * + * @param connectId + * @return + */ + List getDatumSnapshotRequest(String connectId); + + DatumSnapshotRequest getDatumSnapshotRequest(String connectId, String dataServerIP); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/ConnectionsResource.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/ConnectionsResource.java new file mode 100644 index 000000000..f0a7cb00e --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/ConnectionsResource.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.resource; + +import com.alipay.sofa.registry.server.session.connections.ConnectionsService; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import java.util.List; + +@Path("connections") +public class ConnectionsResource { + @Autowired + private ConnectionsService connectionsService; + + @POST + @Path("setMax") + @Produces(MediaType.APPLICATION_JSON) + public Response setMax(@FormParam("connections") int connections) { + ResponseBuilder build = Response.status(Response.Status.OK); + connectionsService.setMaxConnections(connections); + return build.build(); + } + + @GET + @Path("query") + @Produces(MediaType.APPLICATION_JSON) + public Response query() { + ResponseBuilder builder = Response.status(Response.Status.OK); + List connectionsSet = connectionsService.getConnections(); + builder.entity(connectionsSet.toArray()); + return builder.build(); + } +} diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/HealthResource.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/HealthResource.java index 37afbd6c2..648acd736 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/HealthResource.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/HealthResource.java @@ -16,15 +16,23 @@ */ package com.alipay.sofa.registry.server.session.resource; -import com.alipay.sofa.registry.common.model.CommonResponse; -import com.alipay.sofa.registry.server.session.bootstrap.SessionServerBootstrap; -import com.alipay.sofa.registry.server.session.node.RaftClientManager; -import org.springframework.beans.factory.annotation.Autowired; - +import javax.annotation.PostConstruct; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.Response.Status; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.metrics.ReporterUtils; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerBootstrap; +import com.alipay.sofa.registry.server.session.node.RaftClientManager; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; /** * @@ -40,11 +48,28 @@ public class HealthResource { @Autowired private SessionServerBootstrap sessionServerBootstrap; + @PostConstruct + public void init() { + MetricRegistry metrics = new MetricRegistry(); + metrics.register("healthCheck", (Gauge) () -> getHealthCheckResult()); + ReporterUtils.startSlf4jReporter(60, metrics); + } + @GET @Path("check") @Produces(MediaType.APPLICATION_JSON) - public CommonResponse checkHealth() { + public Response checkHealth() { + ResponseBuilder builder = Response.status(Response.Status.OK); + CommonResponse response = getHealthCheckResult(); + builder.entity(response); + if (!response.isSuccess()) { + builder.status(Status.INTERNAL_SERVER_ERROR); + } + + return builder.build(); + } + private CommonResponse getHealthCheckResult() { CommonResponse response; StringBuilder sb = new StringBuilder("SessionServerBoot "); @@ -78,7 +103,6 @@ public CommonResponse checkHealth() { } else { response = CommonResponse.buildFailedResponse(sb.toString()); } - return response; } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionDigestResource.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionDigestResource.java index 25148c948..373617b03 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionDigestResource.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionDigestResource.java @@ -16,29 +16,44 @@ */ package com.alipay.sofa.registry.server.session.resource; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + +import com.alipay.sofa.registry.net.NetUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; + +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.StoreData; import com.alipay.sofa.registry.common.model.store.Subscriber; import com.alipay.sofa.registry.common.model.store.Watcher; +import com.alipay.sofa.registry.metrics.ReporterUtils; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.node.NodeManager; +import com.alipay.sofa.registry.server.session.node.NodeManagerFactory; +import com.alipay.sofa.registry.server.session.node.SessionNodeManager; import com.alipay.sofa.registry.server.session.store.DataStore; import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.store.Watchers; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.CollectionUtils; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.CONNECT_ID_SPLIT; /** * @@ -69,11 +84,26 @@ public class SessionDigestResource { @Autowired private SessionServerConfig sessionServerConfig; - private final static String SUB = "SUB"; + private static final String LOCAL_ADDRESS = NetUtil.getLocalAddress().getHostAddress(); + + private final static String SUB = "SUB"; - private final static String PUB = "PUB"; + private final static String PUB = "PUB"; - private final static String WAT = "WAT"; + private final static String WAT = "WAT"; + + private final static String SESSION = "SESSION"; + + private final static String DATA = "DATA"; + + private final static String META = "META"; + + @PostConstruct + public void init() { + MetricRegistry metrics = new MetricRegistry(); + metrics.register("pushSwitch", (Gauge) () -> getPushSwitch()); + ReporterUtils.startSlf4jReporter(60, metrics); + } @GET @Path("{type}/data/query") @@ -95,8 +125,17 @@ public Map> getSessionDataByDataInfoId(@ @POST @Path("{type}/connect/query") @Produces(MediaType.APPLICATION_JSON) - public Map> getSessionDataByConnectId(List connectIds, + public Map> getSessionDataByConnectId(List queryConnectIds, final @PathParam("type") String type) { + List connectIds = new ArrayList<>(queryConnectIds.size()); + for (String queryConnectId : queryConnectIds) { + String connectId = queryConnectId; + if (!queryConnectId.contains(CONNECT_ID_SPLIT)) { + connectId = connectId + CONNECT_ID_SPLIT + LOCAL_ADDRESS + ":" + + sessionServerConfig.getServerPort(); + } + connectIds.add(connectId); + } Map> serverList = new HashMap<>(); if (connectIds != null) { @@ -105,12 +144,15 @@ public Map> getSessionDataByConnectId(Li Map subMap = sessionInterests.queryByConnectId(connectId); Map watcherMap = sessionWatchers.queryByConnectId(connectId); - Collection publishers = - pubMap != null && !pubMap.isEmpty() ? pubMap.values() : new ArrayList<>(); - Collection subscribers = - subMap != null && !subMap.isEmpty() ? subMap.values() : new ArrayList<>(); - Collection watchers = - watcherMap != null && !watcherMap.isEmpty() ? watcherMap.values() : new ArrayList<>(); + Collection publishers = pubMap != null && !pubMap.isEmpty() + ? pubMap.values() + : new ArrayList<>(); + Collection subscribers = subMap != null && !subMap.isEmpty() + ? subMap.values() + : new ArrayList<>(); + Collection watchers = watcherMap != null && !watcherMap.isEmpty() + ? watcherMap.values() + : new ArrayList<>(); fillServerList(type, serverList, publishers, subscribers, watchers); }); } @@ -142,6 +184,23 @@ public Map getPushSwitch() { return resultMap; } + @GET + @Path("getDataInfoIdList") + @Produces(MediaType.APPLICATION_JSON) + public Collection getDataInfoIdList() { + Collection ret = new HashSet<>(); + ret.addAll(sessionInterests.getInterestDataInfoIds()); + ret.addAll(sessionDataStore.getStoreDataInfoIds()); + return ret; + } + + @GET + @Path("checkSumDataInfoIdList") + @Produces(MediaType.APPLICATION_JSON) + public int checkSumDataInfoIdList() { + return getDataInfoIdList().hashCode(); + } + private void fillServerList(String type, Map> serverList, Collection publishers, @@ -190,4 +249,78 @@ private void fillServerList(String type, } } } + + @GET + @Path("{type}/serverList/query") + @Produces(MediaType.APPLICATION_JSON) + public List getServerListAll(@PathParam("type") String type) { + List serverList = new ArrayList<>(); + if (type != null && !type.isEmpty()) { + String inputType = type.toUpperCase(); + + switch (inputType) { + case SESSION: + serverList = getSessionServerList(); + break; + case DATA: + serverList = getDataServerList(); + break; + case META: + serverList = getMetaServerList(); + break; + default: + serverList = new ArrayList<>(); + break; + } + + } + return serverList; + } + + public List getSessionServerList() { + + List serverList = new ArrayList<>(); + NodeManager nodeManager = NodeManagerFactory.getNodeManager(NodeType.SESSION); + + if (nodeManager instanceof SessionNodeManager) { + SessionNodeManager sessionNodeManager = (SessionNodeManager) nodeManager; + serverList = sessionNodeManager.getZoneServerList(sessionServerConfig + .getSessionServerRegion()); + } + return serverList; + } + + public List getDataServerList() { + List serverList = new ArrayList<>(); + NodeManager dataNodeManager = NodeManagerFactory.getNodeManager(NodeType.DATA); + + Collection dataNodes = dataNodeManager.getDataCenterNodes(); + + if (!CollectionUtils.isEmpty(dataNodes)) { + for (Node dataNode : dataNodes) { + if (dataNode.getNodeUrl() == null || dataNode.getNodeUrl().getIpAddress() == null) { + continue; + } + serverList.add(dataNode.getNodeUrl().getIpAddress()); + } + } + return serverList; + } + + public List getMetaServerList() { + List serverList = new ArrayList<>(); + NodeManager metaNodeManager = NodeManagerFactory.getNodeManager(NodeType.META); + + Collection metaNodes = metaNodeManager.getDataCenterNodes(); + + if (!CollectionUtils.isEmpty(metaNodes)) { + for (Node metaNode : metaNodes) { + if (metaNode.getNodeUrl() == null || metaNode.getNodeUrl().getIpAddress() == null) { + continue; + } + serverList.add(metaNode.getNodeUrl().getIpAddress()); + } + } + return serverList; + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionOpenResource.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionOpenResource.java index f0ce93c22..3d697df23 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionOpenResource.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/resource/SessionOpenResource.java @@ -66,8 +66,8 @@ public List getSessionServerListJson(@QueryParam("zone") String zone) { serverList = sessionNodeManager.getZoneServerList(zone); serverList = serverList.stream() - .map(server -> server + ":" + sessionServerConfig.getServerPort()) - .collect(Collectors.toList()); + .map(server -> server + ":" + sessionServerConfig.getServerPort()) + .collect(Collectors.toList()); } return serverList; diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/ExecutorManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/ExecutorManager.java index cd11a6ae6..f7bc34d69 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/ExecutorManager.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/ExecutorManager.java @@ -16,26 +16,30 @@ */ package com.alipay.sofa.registry.server.session.scheduler; -import com.alipay.sofa.registry.metrics.TaskMetrics; -import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; -import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import com.alipay.sofa.registry.server.session.node.NodeManager; -import com.alipay.sofa.registry.server.session.registry.Registry; -import com.alipay.sofa.registry.task.scheduler.TimedSupervisorTask; -import com.alipay.sofa.registry.util.NamedThreadFactory; -import org.springframework.beans.factory.annotation.Autowired; - import java.util.HashMap; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.remoting.exchange.NodeExchanger; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.node.NodeManager; +import com.alipay.sofa.registry.server.session.registry.Registry; +import com.alipay.sofa.registry.task.scheduler.TimedSupervisorTask; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer.TaskFailedCallback; +import com.alipay.sofa.registry.util.NamedThreadFactory; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + /** * * @author shangyu.wh @@ -43,170 +47,211 @@ */ public class ExecutorManager { - private final ScheduledExecutorService scheduler; + private static final Logger LOGGER = LoggerFactory + .getLogger(ExecutorManager.class); - private final ThreadPoolExecutor fetchDataExecutor; - private final ThreadPoolExecutor standaloneCheckVersionExecutor; - private final ThreadPoolExecutor renNewDataExecutor; - private final ThreadPoolExecutor getSessionNodeExecutor; - private final ThreadPoolExecutor connectMetaExecutor; - private final ThreadPoolExecutor connectDataExecutor; + private final ScheduledThreadPoolExecutor scheduler; - private final ExecutorService checkPushExecutor; - private final ExecutorService pushTaskClosureExecutor; - private final ThreadPoolExecutor accessDataExecutor; - private final ThreadPoolExecutor dataChangeRequestExecutor; - private final ThreadPoolExecutor pushTaskExecutor; - private final ThreadPoolExecutor disconnectClientExecutor; + private final ThreadPoolExecutor defaultRequestExecutor; + private final ThreadPoolExecutor fetchDataExecutor; + private final ThreadPoolExecutor standaloneCheckVersionExecutor; + private final ThreadPoolExecutor renNewDataExecutor; + private final ThreadPoolExecutor getSessionNodeExecutor; + private final ThreadPoolExecutor connectMetaExecutor; + private final ThreadPoolExecutor connectDataExecutor; - private SessionServerConfig sessionServerConfig; + private final ExecutorService checkPushExecutor; + private final ThreadPoolExecutor accessDataExecutor; + private final ThreadPoolExecutor dataChangeRequestExecutor; + private final ThreadPoolExecutor pushTaskExecutor; + private final ThreadPoolExecutor connectClientExecutor; + private final ThreadPoolExecutor publishDataExecutor; + private final ThreadPoolExecutor cleanInvalidClientExecutor; + + private final AsyncHashedWheelTimer pushTaskCheckAsyncHashedWheelTimer; + + private SessionServerConfig sessionServerConfig; @Autowired - private Registry sessionRegistry; + private Registry sessionRegistry; @Autowired - private NodeManager sessionNodeManager; + private NodeManager sessionNodeManager; @Autowired - private NodeManager dataNodeManager; + private NodeManager dataNodeManager; @Autowired - private NodeManager metaNodeManager; + private NodeManager metaNodeManager; @Autowired - protected NodeExchanger metaNodeExchanger; + protected NodeExchanger metaNodeExchanger; @Autowired - private NodeExchanger dataNodeExchanger; + private NodeExchanger dataNodeExchanger; + + private static final String DEFAULT_REQUEST_EXECUTOR = "DefaultRequestExecutor"; - private Map reportExecutors = new HashMap<>(); + private static final String PUSH_TASK_EXECUTOR = "PushTaskExecutor"; - private static final String PUSH_TASK_EXECUTOR = "PushTaskExecutor"; + private static final String ACCESS_DATA_EXECUTOR = "AccessDataExecutor"; - private static final String ACCESS_DATA_EXECUTOR = "AccessDataExecutor"; + private static final String DATA_CHANGE_REQUEST_EXECUTOR = "DataChangeRequestExecutor"; - private static final String DATA_CHANGE_REQUEST_EXECUTOR = "DataChangeRequestExecutor"; + private static final String USER_DATA_ELEMENT_PUSH_TASK_CHECK_EXECUTOR = "UserDataElementPushCheckExecutor"; - private static final String USER_DATA_ELEMENT_PUSH_TASK_CHECK_EXECUTOR = "UserDataElementPushCheckExecutor"; + private static final String PUSH_TASK_CLOSURE_CHECK_EXECUTOR = "PushTaskClosureCheckExecutor"; - private static final String PUSH_TASK_CLOSURE_CHECK_EXECUTOR = "PushTaskClosureCheckExecutor"; + private static final String CONNECT_CLIENT_EXECUTOR = "ConnectClientExecutor"; - private static final String DISCONNECT_CLIENT_EXECUTOR = "DisconnectClientExecutor"; + private static final String PUBLISH_DATA_EXECUTOR = "PublishDataExecutor"; public ExecutorManager(SessionServerConfig sessionServerConfig) { this.sessionServerConfig = sessionServerConfig; - scheduler = new ScheduledThreadPoolExecutor(7, new NamedThreadFactory("SessionScheduler")); + scheduler = new ScheduledThreadPoolExecutor( + sessionServerConfig.getSessionSchedulerPoolSize(), + new NamedThreadFactory("SessionScheduler")); fetchDataExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-fetchData")); + new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-fetchData")); renNewDataExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-reNewData")); + new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-renewData")); getSessionNodeExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-getSessionNode")); + new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-getSessionNode")); standaloneCheckVersionExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-standaloneCheckVersion")); + new SynchronousQueue<>(), + new NamedThreadFactory("SessionScheduler-standaloneCheckVersion")); connectMetaExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-connectMetaServer")); + new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-connectMetaServer")); connectDataExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, - new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-connectDataServer")); - - accessDataExecutor = reportExecutors - .computeIfAbsent(ACCESS_DATA_EXECUTOR, k -> new SessionThreadPoolExecutor(ACCESS_DATA_EXECUTOR, - sessionServerConfig.getAccessDataExecutorMinPoolSize(), - sessionServerConfig.getAccessDataExecutorMaxPoolSize(), - sessionServerConfig.getAccessDataExecutorKeepAliveTime(), - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(sessionServerConfig.getAccessDataExecutorQueueSize()), - new NamedThreadFactory("AccessData-executor", true))); - - pushTaskExecutor = reportExecutors.computeIfAbsent(PUSH_TASK_EXECUTOR, - k -> new ThreadPoolExecutor(sessionServerConfig.getPushTaskExecutorMinPoolSize(), - sessionServerConfig.getPushTaskExecutorMaxPoolSize(), - sessionServerConfig.getPushTaskExecutorKeepAliveTime(), TimeUnit.SECONDS, - new LinkedBlockingQueue<>(sessionServerConfig.getPushTaskExecutorQueueSize()), - new NamedThreadFactory("PushTask-executor", true))); - - TaskMetrics.getInstance().registerThreadExecutor(PUSH_TASK_EXECUTOR, pushTaskExecutor); - - dataChangeRequestExecutor = reportExecutors.computeIfAbsent(DATA_CHANGE_REQUEST_EXECUTOR, - k -> new SessionThreadPoolExecutor(DATA_CHANGE_REQUEST_EXECUTOR, - sessionServerConfig.getDataChangeExecutorMinPoolSize(), - sessionServerConfig.getDataChangeExecutorMaxPoolSize(), - sessionServerConfig.getDataChangeExecutorKeepAliveTime(), - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(sessionServerConfig.getDataChangeExecutorQueueSize()), - new NamedThreadFactory( - "DataChangeRequestHandler-executor", true))); - - checkPushExecutor = reportExecutors - .computeIfAbsent(USER_DATA_ELEMENT_PUSH_TASK_CHECK_EXECUTOR, k -> new SessionThreadPoolExecutor( - USER_DATA_ELEMENT_PUSH_TASK_CHECK_EXECUTOR, 100, 600, 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue(100000), - new NamedThreadFactory("UserDataElementPushCheck-executor", true))); - - pushTaskClosureExecutor = reportExecutors - .computeIfAbsent(PUSH_TASK_CLOSURE_CHECK_EXECUTOR, k -> new SessionThreadPoolExecutor( - PUSH_TASK_CLOSURE_CHECK_EXECUTOR, 80, 400, 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue(10000), - new NamedThreadFactory("PushTaskClosureCheck", true))); - - disconnectClientExecutor = reportExecutors.computeIfAbsent(DISCONNECT_CLIENT_EXECUTOR,k->new SessionThreadPoolExecutor( - DISCONNECT_CLIENT_EXECUTOR, sessionServerConfig.getDisconnectClientExecutorMinPoolSize(), - sessionServerConfig.getDisconnectClientExecutorMaxPoolSize(), 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue(sessionServerConfig.getDisconnectClientExecutorQueueSize()), - new NamedThreadFactory("DisconnectClientExecutor", true))); - + new SynchronousQueue<>(), new NamedThreadFactory("SessionScheduler-connectDataServer")); + + cleanInvalidClientExecutor = new ThreadPoolExecutor(1, 2/*CONFIG*/, 0, TimeUnit.SECONDS, + new SynchronousQueue<>(), + new NamedThreadFactory("SessionScheduler-cleanInvalidClient")); + + defaultRequestExecutor = new SessionMetricsThreadPoolExecutor(DEFAULT_REQUEST_EXECUTOR, + sessionServerConfig.getDefaultRequestExecutorMinPoolSize(), + sessionServerConfig.getDefaultRequestExecutorMaxPoolSize(), + sessionServerConfig.getDefaultRequestExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(sessionServerConfig.getDefaultRequestExecutorQueueSize()), + new NamedThreadFactory(DEFAULT_REQUEST_EXECUTOR, true)); + + accessDataExecutor = new SessionMetricsThreadPoolExecutor(ACCESS_DATA_EXECUTOR, + sessionServerConfig.getAccessDataExecutorMinPoolSize(), + sessionServerConfig.getAccessDataExecutorMaxPoolSize(), + sessionServerConfig.getAccessDataExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(sessionServerConfig.getAccessDataExecutorQueueSize()), + new NamedThreadFactory("AccessData-executor", true), (r, executor) -> { + String msg = String.format( + "Task(%s) %s rejected from %s, just ignore it to let client timeout.", + r.getClass(), r, executor); + LOGGER.error(msg); + }); + + pushTaskExecutor = new SessionMetricsThreadPoolExecutor(PUSH_TASK_EXECUTOR, + sessionServerConfig.getPushTaskExecutorMinPoolSize(), + sessionServerConfig.getPushTaskExecutorMaxPoolSize(), + sessionServerConfig.getPushTaskExecutorKeepAliveTime(), TimeUnit.SECONDS, + new LinkedBlockingQueue<>(sessionServerConfig.getPushTaskExecutorQueueSize()), + new NamedThreadFactory("PushTask-executor", true), + new ThreadPoolExecutor.AbortPolicy()); + + dataChangeRequestExecutor = new SessionMetricsThreadPoolExecutor( + DATA_CHANGE_REQUEST_EXECUTOR, sessionServerConfig.getDataChangeExecutorMinPoolSize(), + sessionServerConfig.getDataChangeExecutorMaxPoolSize(), + sessionServerConfig.getDataChangeExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(sessionServerConfig.getDataChangeExecutorQueueSize()), + new NamedThreadFactory("DataChangeRequestHandler-executor", true)); + + checkPushExecutor = new SessionMetricsThreadPoolExecutor( + USER_DATA_ELEMENT_PUSH_TASK_CHECK_EXECUTOR, 100, 600, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue(150000), + new NamedThreadFactory("UserDataElementPushCheck-executor", true)); + + connectClientExecutor = new SessionMetricsThreadPoolExecutor(CONNECT_CLIENT_EXECUTOR, + sessionServerConfig.getConnectClientExecutorMinPoolSize(), + sessionServerConfig.getConnectClientExecutorMaxPoolSize(), 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue(sessionServerConfig.getConnectClientExecutorQueueSize()), + new NamedThreadFactory("DisconnectClientExecutor", true)); + + pushTaskCheckAsyncHashedWheelTimer = new AsyncHashedWheelTimer( + new NamedThreadFactory("PushTaskConfirmCheck-executor", true), + sessionServerConfig.getPushTaskConfirmCheckWheelTicksDuration(), TimeUnit.MILLISECONDS, + sessionServerConfig.getPushTaskConfirmCheckWheelTicksSize(), + sessionServerConfig.getPushTaskConfirmCheckExecutorThreadSize(), + sessionServerConfig.getPushTaskConfirmCheckExecutorQueueSize(), + new ThreadFactoryBuilder().setNameFormat("PushTaskConfirmCheck-executor-%d").build(), + new TaskFailedCallback() { + @Override + public void executionRejected(Throwable e) { + LOGGER.error("executionRejected: " + e.getMessage(), e); + } + + @Override + public void executionFailed(Throwable e) { + LOGGER.error("executionFailed: " + e.getMessage(), e); + } + }); + publishDataExecutor = new SessionMetricsThreadPoolExecutor(PUBLISH_DATA_EXECUTOR, + sessionServerConfig.getPublishDataExecutorMinPoolSize(), + sessionServerConfig.getPublishDataExecutorMaxPoolSize(), + sessionServerConfig.getPublishDataExecutorKeepAliveTime(), TimeUnit.SECONDS, + new ArrayBlockingQueue<>(sessionServerConfig.getPublishDataExecutorQueueSize()), + new NamedThreadFactory("PublishData-executor", true)); } public void startScheduler() { scheduler.schedule( - new TimedSupervisorTask("FetchData", scheduler, fetchDataExecutor, - sessionServerConfig.getSchedulerFetchDataTimeout(), TimeUnit.MINUTES, - sessionServerConfig.getSchedulerFetchDataExpBackOffBound(), - () -> sessionRegistry.fetchChangData()), - sessionServerConfig.getSchedulerFetchDataFirstDelay(), TimeUnit.SECONDS); + new TimedSupervisorTask("FetchData", scheduler, fetchDataExecutor, + sessionServerConfig.getSchedulerFetchDataTimeout(), TimeUnit.MINUTES, + sessionServerConfig.getSchedulerFetchDataExpBackOffBound(), + () -> sessionRegistry.fetchChangData()), + sessionServerConfig.getSchedulerFetchDataFirstDelay(), TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("ReNewData", scheduler, renNewDataExecutor, - sessionServerConfig.getSchedulerHeartbeatTimeout(), TimeUnit.SECONDS, - sessionServerConfig.getSchedulerHeartbeatExpBackOffBound(), - () -> sessionNodeManager.reNewNode()), - sessionServerConfig.getSchedulerHeartbeatFirstDelay(), TimeUnit.SECONDS); + new TimedSupervisorTask("RenewData", scheduler, renNewDataExecutor, + sessionServerConfig.getSchedulerHeartbeatTimeout(), TimeUnit.SECONDS, + sessionServerConfig.getSchedulerHeartbeatExpBackOffBound(), + () -> sessionNodeManager.renewNode()), + sessionServerConfig.getSchedulerHeartbeatFirstDelay(), TimeUnit.SECONDS); + + scheduler + .schedule(new TimedSupervisorTask("GetSessionNode", scheduler, getSessionNodeExecutor, + sessionServerConfig.getSchedulerGetSessionNodeTimeout(), TimeUnit.SECONDS, + sessionServerConfig.getSchedulerGetSessionNodeExpBackOffBound(), () -> { + sessionNodeManager.getAllDataCenterNodes(); + dataNodeManager.getAllDataCenterNodes(); + metaNodeManager.getAllDataCenterNodes(); + }), sessionServerConfig.getSchedulerGetSessionNodeFirstDelay(), TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("GetSessionNode", scheduler, getSessionNodeExecutor, - sessionServerConfig.getSchedulerGetSessionNodeTimeout(), TimeUnit.SECONDS, - sessionServerConfig.getSchedulerGetSessionNodeExpBackOffBound(), - () -> { - sessionNodeManager.getAllDataCenterNodes(); - dataNodeManager.getAllDataCenterNodes(); - metaNodeManager.getAllDataCenterNodes(); - }), - sessionServerConfig.getSchedulerGetSessionNodeFirstDelay(), TimeUnit.SECONDS); + new TimedSupervisorTask("ConnectMetaServer", scheduler, connectMetaExecutor, + sessionServerConfig.getSchedulerConnectMetaTimeout(), TimeUnit.SECONDS, + sessionServerConfig.getSchedulerConnectMetaExpBackOffBound(), + () -> metaNodeExchanger.connectServer()), + sessionServerConfig.getSchedulerConnectMetaFirstDelay(), TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("ConnectMetaServer", scheduler, connectMetaExecutor, - sessionServerConfig.getSchedulerConnectMetaTimeout(), TimeUnit.SECONDS, - sessionServerConfig.getSchedulerConnectMetaExpBackOffBound(), - () -> metaNodeExchanger.connectServer()), - sessionServerConfig.getSchedulerConnectMetaFirstDelay(), TimeUnit.SECONDS); + new TimedSupervisorTask("ConnectDataServer", scheduler, connectDataExecutor, + sessionServerConfig.getSchedulerConnectDataTimeout(), TimeUnit.SECONDS, + sessionServerConfig.getSchedulerConnectDataExpBackOffBound(), + () -> dataNodeExchanger.connectServer()), + sessionServerConfig.getSchedulerConnectDataFirstDelay(), TimeUnit.SECONDS); scheduler.schedule( - new TimedSupervisorTask("ConnectDataServer", scheduler, connectDataExecutor, - sessionServerConfig.getSchedulerConnectDataTimeout(), TimeUnit.SECONDS, - sessionServerConfig.getSchedulerConnectDataExpBackOffBound(), - () -> dataNodeExchanger.connectServer()), - sessionServerConfig.getSchedulerConnectDataFirstDelay(), TimeUnit.SECONDS); + new TimedSupervisorTask("CleanInvalidClient", scheduler, cleanInvalidClientExecutor, + sessionServerConfig.getSchedulerCleanInvalidClientTimeOut(), TimeUnit.MINUTES, + sessionServerConfig.getSchedulerCleanInvalidClientBackOffBound(), + () -> sessionRegistry.cleanClientConnect()), + sessionServerConfig.getSchedulerCleanInvalidClientFirstDelay(), TimeUnit.MINUTES); } public void stopScheduler() { @@ -253,19 +298,15 @@ public void stopScheduler() { dataChangeRequestExecutor.shutdown(); } - if (pushTaskClosureExecutor != null && !pushTaskClosureExecutor.isShutdown()) { - pushTaskClosureExecutor.shutdown(); + if (connectClientExecutor != null && !connectClientExecutor.isShutdown()) { + connectClientExecutor.shutdown(); } - if (disconnectClientExecutor != null && !disconnectClientExecutor.isShutdown()) { - disconnectClientExecutor.shutdown(); + if (publishDataExecutor != null && !publishDataExecutor.isShutdown()) { + publishDataExecutor.shutdown(); } } - public Map getReportExecutors() { - return reportExecutors; - } - public ThreadPoolExecutor getAccessDataExecutor() { return accessDataExecutor; } @@ -282,12 +323,19 @@ public ThreadPoolExecutor getDataChangeRequestExecutor() { return dataChangeRequestExecutor; } - public ExecutorService getPushTaskClosureExecutor() { - return pushTaskClosureExecutor; + public ThreadPoolExecutor getConnectClientExecutor() { + return connectClientExecutor; } - public ThreadPoolExecutor getDisconnectClientExecutor() { - return disconnectClientExecutor; + public AsyncHashedWheelTimer getPushTaskCheckAsyncHashedWheelTimer() { + return pushTaskCheckAsyncHashedWheelTimer; } + public ThreadPoolExecutor getPublishDataExecutor() { + return publishDataExecutor; + } + + public ThreadPoolExecutor getDefaultRequestExecutor() { + return defaultRequestExecutor; + } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionMetricsThreadPoolExecutor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionMetricsThreadPoolExecutor.java new file mode 100644 index 000000000..7e52901b3 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/SessionMetricsThreadPoolExecutor.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.metrics.TaskMetrics; + +/** + * + * @author shangyu.wh + * @version $Id: ThreadPoolExecutorSession.java, v 0.1 2018-10-11 19:07 shangyu.wh Exp $ + */ +public class SessionMetricsThreadPoolExecutor extends ThreadPoolExecutor { + + private static final Logger LOGGER = LoggerFactory + .getLogger(SessionMetricsThreadPoolExecutor.class); + + private String executorName; + + public SessionMetricsThreadPoolExecutor(String executorName, int corePoolSize, + int maximumPoolSize, long keepAliveTime, TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + this.executorName = "Session-" + executorName; + registerTaskMetrics(); + this.setRejectedExecutionHandler(handler); + } + + public SessionMetricsThreadPoolExecutor(String executorName, int corePoolSize, + int maximumPoolSize, long keepAliveTime, TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { + this(executorName, corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, + threadFactory, (r, executor) -> { + String msg = String.format( + "Task(%s) %s rejected from %s, throw RejectedExecutionException.", r.getClass(), + r, executor); + LOGGER.error(msg); + throw new RejectedExecutionException(msg); + }); + } + + private void registerTaskMetrics() { + TaskMetrics.getInstance().registerThreadExecutor(executorName, this); + } + + @Override + public String toString() { + return (new StringBuilder(executorName).append(" ").append(super.toString())).toString(); + } + +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/AbstractSessionTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/AbstractSessionTask.java index 3135c1461..ae41f743e 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/AbstractSessionTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/AbstractSessionTask.java @@ -16,16 +16,17 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.task.Retryable; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - /** * * @author shangyu.wh + * @author kezhu.wukz * @version $Id: AbstractSessionTask.java, v 0.1 2018-01-15 14:35 shangyu.wh Exp $ */ public abstract class AbstractSessionTask implements SessionTask, Retryable { @@ -46,6 +47,10 @@ public synchronized String getTaskId() { return taskId; } + protected synchronized void setTaskId(String taskId) { + this.taskId = taskId; + } + protected boolean checkRetryTimes(int configTimes) { if (configTimes > 0) { if (execCount.incrementAndGet() > configTimes) { @@ -58,6 +63,10 @@ protected boolean checkRetryTimes(int configTimes) { return false; } + protected int getExecCount() { + return execCount.get(); + } + @Override public long getExpiryTime() { return -1; diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/CancelDataTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/CancelDataTask.java index df380c58b..f07a089e2 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/CancelDataTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/CancelDataTask.java @@ -16,6 +16,8 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.util.List; + import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.DataNodeService; import com.alipay.sofa.registry.server.session.store.DataStore; @@ -23,9 +25,6 @@ import com.alipay.sofa.registry.server.session.store.Watchers; import com.alipay.sofa.registry.task.listener.TaskEvent; -import java.util.ArrayList; -import java.util.List; - /** * * @author shangyu.wh @@ -61,31 +60,23 @@ public CancelDataTask(Interests sessionInterests, DataStore sessionDataStore, @Override public void execute() { - if (connectIds.isEmpty()) { - throw new IllegalArgumentException("Input clientOff connectIds error!"); - } - - //remove local first,data node send error depend on other task check - List connectIdsPub = new ArrayList<>(); - for (String connectId : connectIds) { - - if (sessionDataStore.deleteByConnectId(connectId)) { - connectIdsPub.add(connectId); - } - sessionInterests.deleteByConnectId(connectId); - - sessionWatchers.deleteByConnectId(connectId); - } - - dataNodeService.clientOff(connectIdsPub); + dataNodeService.clientOff(connectIds); } @Override public void setTaskEvent(TaskEvent taskEvent) { - Object obj = taskEvent.getEventObj(); + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); if (obj instanceof List) { this.connectIds = (List) obj; + if (connectIds.isEmpty()) { + throw new IllegalArgumentException("Input clientOff connectIds error!"); + } } else { throw new IllegalArgumentException("Input task event object error!"); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchCloudTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchCloudTask.java index d6a423adc..e893179cd 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchCloudTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchCloudTask.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; + import com.alipay.sofa.registry.common.model.Node.NodeType; import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.store.BaseInfo.ClientVersion; @@ -26,6 +35,7 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.cache.CacheAccessException; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.cache.DatumKey; import com.alipay.sofa.registry.server.session.cache.Key; @@ -42,15 +52,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; - /** * * @author shangyu.wh @@ -98,6 +99,11 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof String)) { @@ -146,14 +152,19 @@ public void execute() { } public PushTaskClosure getTaskClosure(Map datumMap) { - PushTaskClosure pushTaskClosure = new PushTaskClosure(executorManager.getPushTaskClosureExecutor()); + + PushTaskClosure pushTaskClosure = new PushTaskClosure(executorManager.getPushTaskCheckAsyncHashedWheelTimer(), + sessionServerConfig, fetchDataInfoId); pushTaskClosure.setTaskClosure((status, task) -> { if (status == ProcessingResult.Success) { if (sessionServerConfig.isStopPushSwitch()) { LOGGER.info("Stop Push switch on,dataInfoId {} version can not be update!", fetchDataInfoId); return; } + List dataCenters = sessionInterests.getDataCenters(); datumMap.forEach((dataCenter, datum) -> { + dataCenters.remove(dataCenter); + String dataInfoId = fetchDataInfoId; Long version = datum.getVersion(); boolean result = sessionInterests.checkAndUpdateInterestVersions(dataCenter, dataInfoId, version); @@ -163,10 +174,20 @@ public PushTaskClosure getTaskClosure(Map datumMap) } else { LOGGER.info( "Push all tasks success,but dataCenter:{} dataInfoId:{} version:{} need not update!", - dataCenter, - dataInfoId, version); + dataCenter, dataInfoId, version); } }); + + // now, elements left in dataCenters, means that datum of the dataCenter, + // cannot be obtained from DataServer this time, it may cause pushing empty datum wrongly, + // so it is necessary to ensure that it can be checked later + dataCenters.forEach(dataCenter -> { + boolean result = sessionInterests.checkAndUpdateInterestVersionZero(dataCenter, fetchDataInfoId); + LOGGER.warn( + "Obtained datum from DataServer({}) failed, set sessionInterests dataInfoId({}) version zero, return {}", + dataCenter, fetchDataInfoId, result); + }); + } else { LOGGER.warn("Push tasks found error,subscribers version can not be update!dataInfoId={}", fetchDataInfoId); @@ -199,7 +220,12 @@ private Map getDatumsCache() { new DatumKey(fetchDataInfoId, dataCenter))). collect(Collectors.toList()); - Map values = sessionCacheService.getValues(keys); + Map values = null; + try { + values = sessionCacheService.getValues(keys); + } catch (CacheAccessException e) { + LOGGER.error(String.format("error when access cache: %s", e.getMessage()), e); + } if (values != null) { values.forEach((key, value) -> { @@ -239,6 +265,7 @@ private void fireReceiveDataPushTask(Map datums, Map subscriberMap, PushTaskClosure pushTaskClosure) { Collection subscribers = new ArrayList<>(subscriberMap.values()); + LOGGER.info("Datums push={}", datums); ReceivedData receivedData = ReceivedDataConverter.getReceivedDataMulti(datums, scopeEnum, subscriberRegisterIdList, subscriber); @@ -248,8 +275,10 @@ private void fireReceiveDataPushTask(Map datums, TaskEvent taskEvent = new TaskEvent(parameter, TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); taskEvent.setTaskClosure(pushTaskClosure); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); - taskLogger.info("send {} taskURL:{},taskScope:{}", taskEvent.getTaskType(), - subscriber.getSourceAddress(), scopeEnum); + + taskLogger.info("send {} taskURL:{},taskScope:{},version:{},taskId={}", + taskEvent.getTaskType(), subscriber.getSourceAddress(), scopeEnum, + receivedData.getVersion(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -267,9 +296,10 @@ private void fireUserDataElementPushTask(Datum datum, Subscriber subscriber, int size = datum != null && datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={}", + taskLogger.info( + "send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={},taskId={}", taskEvent.getTaskType(), subscriber.getSourceAddress(), datum.getDataInfoId(), - datum.getDataCenter(), size, subscribers.size()); + datum.getDataCenter(), size, subscribers.size(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -286,9 +316,10 @@ private void fireUserDataElementMultiPushTask(Datum datum, Subscriber subscriber int size = datum != null && datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={}", + taskLogger.info( + "send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={},taskId={}", taskEvent.getTaskType(), subscriber.getSourceAddress(), datum.getDataInfoId(), - datum.getDataCenter(), size, subscribers.size()); + datum.getDataCenter(), size, subscribers.size(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchTask.java index 3757a3117..505b13429 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataChangeFetchTask.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Predicate; + import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.sessionserver.DataChangeRequest; import com.alipay.sofa.registry.common.model.store.BaseInfo.ClientVersion; @@ -26,6 +35,7 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.cache.CacheAccessException; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.cache.DatumKey; import com.alipay.sofa.registry.server.session.cache.Key; @@ -39,15 +49,7 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; - -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.Predicate; +import com.alipay.sofa.registry.util.DatumVersionUtil; /** * @@ -98,11 +100,16 @@ public void execute() { Datum datum = getDatumCache(); if (datum != null) { - PushTaskClosure pushTaskClosure = getTaskClosure(); + PushTaskClosure pushTaskClosure = getTaskClosure(datum.getVersion()); for (ScopeEnum scopeEnum : ScopeEnum.values()) { Map> map = getCache(scopeEnum); if (map != null && !map.isEmpty()) { + LOGGER + .info( + "Get all subscribers to send from cache size:{},which dataInfoId:{} on dataCenter:{},scope:{}", + map.size(), dataChangeRequest.getDataInfoId(), + dataChangeRequest.getDataCenter(), scopeEnum); for (Entry> entry : map.entrySet()) { Map subscriberMap = entry.getValue(); if (subscriberMap != null && !subscriberMap.isEmpty()) { @@ -181,33 +188,35 @@ private Collection subscribersVersionCheck(Collection su return subscribersSend; } - public PushTaskClosure getTaskClosure() { + public PushTaskClosure getTaskClosure(Long version) { //this for all this dataInfoId push result get and call back to change version - PushTaskClosure pushTaskClosure = new PushTaskClosure(executorManager.getPushTaskClosureExecutor()); + PushTaskClosure pushTaskClosure = new PushTaskClosure(executorManager.getPushTaskCheckAsyncHashedWheelTimer(), + sessionServerConfig, dataChangeRequest.getDataInfoId()); pushTaskClosure.setTaskClosure((status, task) -> { String dataCenter = dataChangeRequest.getDataCenter(); String dataInfoId = dataChangeRequest.getDataInfoId(); - Long version = dataChangeRequest.getVersion(); + Long changeVersion = dataChangeRequest.getVersion(); if (status == ProcessingResult.Success) { if (sessionServerConfig.isStopPushSwitch()) { - LOGGER.info("Stop Push switch on,dataCenter {} dataInfoId {} version {} can not be update!", - dataCenter, dataInfoId, version); + LOGGER.info("Stop Push switch on, dataCenter:{}, dataInfoId:{}, changeVersion:{}, pushVersion:{}, can not be update!", + dataCenter, dataInfoId, changeVersion, version); return; } boolean result = sessionInterests.checkAndUpdateInterestVersions(dataCenter, dataInfoId, version); if (result) { - LOGGER.info("Push all tasks success,dataCenter:{} dataInfoId:{} version:{} update!", dataCenter, - dataInfoId, version); + LOGGER.info("Push all tasks success, dataCenter:{}, dataInfoId:{}, changeVersion:{}, pushVersion:{}, update!", dataCenter, + dataInfoId, changeVersion, version); } else { LOGGER.info("Push all tasks success,but dataCenter:{} dataInfoId:{} version:{} need not update!", - dataCenter, - dataInfoId, version); + dataCenter, dataInfoId, version); + LOGGER.info("Push all tasks success, but dataCenter:{}, dataInfoId:{}, changeVersion:{}, pushVersion:{}, need not update!", + dataCenter, dataInfoId, changeVersion, version); } } else { LOGGER.warn( - "Push tasks found error,subscribers version can not be update!dataCenter:{} dataInfoId:{} version:{}", - dataCenter, dataInfoId, version); + "Push tasks found error, subscribers version can not be update! dataCenter:{}, dataInfoId:{}, changeVersion:{}, pushVersion:{}", + dataCenter, dataInfoId, changeVersion, version); } }); return pushTaskClosure; @@ -221,26 +230,27 @@ private void evictReSubscribers(Collection subscribersPush) { } private void fireReceivedDataMultiPushTask(Datum datum, List subscriberRegisterIdList, - Collection subscribers, - ScopeEnum scopeEnum, Subscriber subscriber, - PushTaskClosure pushTaskClosure) { + Collection subscribers, ScopeEnum scopeEnum, + Subscriber subscriber, PushTaskClosure pushTaskClosure) { String dataId = datum.getDataId(); + String clientCell = sessionServerConfig.getClientCell(subscriber.getCell()); Predicate zonePredicate = (zone) -> { - if (!sessionServerConfig.getSessionServerRegion().equals(zone)) { + if (!clientCell.equals(zone)) { if (ScopeEnum.zone == scopeEnum) { // zone scope subscribe only return zone list return true; - } else if (ScopeEnum.dataCenter == scopeEnum) { + } else if (ScopeEnum.dataCenter == scopeEnum || ScopeEnum.global == scopeEnum) { // disable zone config - return sessionServerConfig.isInvalidForeverZone(zone) - && !sessionServerConfig.isInvalidIgnored(dataId); + return sessionServerConfig.isInvalidForeverZone(zone) && !sessionServerConfig + .isInvalidIgnored(dataId); } } return false; }; - ReceivedData receivedData = ReceivedDataConverter.getReceivedDataMulti(datum, scopeEnum, - subscriberRegisterIdList, sessionServerConfig.getSessionServerRegion(), zonePredicate); + ReceivedData receivedData = ReceivedDataConverter + .getReceivedDataMulti(datum, scopeEnum, subscriberRegisterIdList, + clientCell, zonePredicate); //trigger push to client node Map parameter = new HashMap<>(); @@ -248,8 +258,8 @@ private void fireReceivedDataMultiPushTask(Datum datum, List subscriberR TaskEvent taskEvent = new TaskEvent(parameter, TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); taskEvent.setTaskClosure(pushTaskClosure); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); - taskLogger.info("send {} taskURL:{},taskScope:{}", taskEvent.getTaskType(), subscriber.getSourceAddress(), - scopeEnum); + taskLogger.info("send {} taskURL:{},taskScope:{},,taskId={}", taskEvent.getTaskType(), + subscriber.getSourceAddress(), scopeEnum, taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -258,10 +268,19 @@ private Map> getCache(ScopeEnum scope } private Datum getDatumCache() { + // build key DatumKey datumKey = new DatumKey(dataChangeRequest.getDataInfoId(), dataChangeRequest.getDataCenter()); - Key key = new Key(KeyType.OBJ, datumKey.getClass().getName(), datumKey); - Value value = sessionCacheService.getValue(key); + Key key = new Key(KeyType.OBJ, DatumKey.class.getName(), datumKey); + + // get from cache (it will fetch from backend server) + Value value = null; + try { + value = sessionCacheService.getValue(key); + } catch (CacheAccessException e) { + LOGGER.error(String.format("error when access cache: %s", e.getMessage()), e); + } + return value == null ? null : value.getPayload(); } @@ -271,16 +290,17 @@ private void fireUserDataElementPushTask(InetSocketAddress address, Datum datum, TaskEvent taskEvent = new TaskEvent(TaskType.USER_DATA_ELEMENT_PUSH_TASK); taskEvent.setTaskClosure(pushTaskClosure); - taskEvent.setSendTimeStamp(datum.getVersion()); + taskEvent.setSendTimeStamp(DatumVersionUtil.getRealTimestamp(datum.getVersion())); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); taskEvent.setAttribute(Constant.PUSH_CLIENT_URL, new URL(address)); int size = datum != null && datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={}", + taskLogger.info( + "send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={},taskId={}", taskEvent.getTaskType(), address, datum.getDataInfoId(), datum.getDataCenter(), size, - subscribers.size()); + subscribers.size(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -290,16 +310,17 @@ private void fireUserDataElementMultiPushTask(InetSocketAddress address, Datum d TaskEvent taskEvent = new TaskEvent(TaskType.USER_DATA_ELEMENT_MULTI_PUSH_TASK); taskEvent.setTaskClosure(pushTaskClosure); - taskEvent.setSendTimeStamp(datum.getVersion()); + taskEvent.setSendTimeStamp(DatumVersionUtil.getRealTimestamp(datum.getVersion())); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); taskEvent.setAttribute(Constant.PUSH_CLIENT_URL, new URL(address)); int size = datum != null && datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={}", + taskLogger.info( + "send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},subSize={},taskId={}", taskEvent.getTaskType(), address, datum.getDataInfoId(), datum.getDataCenter(), size, - subscribers.size()); + subscribers.size(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -310,6 +331,11 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof DataChangeRequest)) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataPushTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataPushTask.java index e372dc21c..fbe8aa852 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataPushTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DataPushTask.java @@ -16,6 +16,15 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Predicate; + import com.alipay.sofa.registry.common.model.dataserver.Datum; import com.alipay.sofa.registry.common.model.sessionserver.DataPushRequest; import com.alipay.sofa.registry.common.model.store.BaseInfo.ClientVersion; @@ -29,20 +38,10 @@ import com.alipay.sofa.registry.server.session.converter.ReceivedDataConverter; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.store.Interests; -import com.alipay.sofa.registry.task.batcher.TaskProcessor.ProcessingResult; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.Predicate; - /** * * @author shangyu.wh @@ -89,8 +88,6 @@ public void execute() { String dataInfoId = datum.getDataInfoId(); - PushTaskClosure pushTaskClosure = getTaskClosure(); - for (ScopeEnum scopeEnum : ScopeEnum.values()) { Map> map = getCache(scopeEnum, dataInfoId); @@ -113,11 +110,11 @@ public void execute() { if (ifLocalDataCenter) { if (isOldVersion) { fireUserDataElementPushTask(entry.getKey(), datum, - subscribersSend, pushTaskClosure); + subscribersSend); } else { fireReceivedDataMultiPushTask(datum, subscriberRegisterIdList, ScopeEnum.zone, - subscriber, pushTaskClosure, subscriberMap); + subscriber, subscriberMap); } } break; @@ -125,17 +122,17 @@ public void execute() { if (ifLocalDataCenter) { if (isOldVersion) { fireUserDataElementMultiPushTask(entry.getKey(), datum, - subscribersSend, pushTaskClosure); + subscribersSend); } else { fireReceivedDataMultiPushTask(datum, subscriberRegisterIdList, scopeEnum, subscriber, - pushTaskClosure, subscriberMap); + subscriberMap); } } break; case global: fireReceivedDataMultiPushTask(datum, subscriberRegisterIdList, - scopeEnum, subscriber, pushTaskClosure, subscriberMap); + scopeEnum, subscriber, subscriberMap); break; default: LOGGER.warn("unknown scope, {}", subscriber); @@ -144,47 +141,21 @@ public void execute() { } } } - pushTaskClosure.start(); } } - public PushTaskClosure getTaskClosure() { - PushTaskClosure pushTaskClosure = new PushTaskClosure(executorManager.getPushTaskClosureExecutor()); - pushTaskClosure.setTaskClosure((status, task) -> { - if (status == ProcessingResult.Success) { - Datum datum = dataPushRequest.getDatum(); - String dataCenter = datum.getDataCenter(); - String dataInfoId = datum.getDataInfoId(); - Long version = datum.getVersion(); - - if (sessionServerConfig.isStopPushSwitch()) { - LOGGER.info("Stop Push switch on,dataCenter {} dataInfoId {} version {} can not be update!", - dataCenter, dataInfoId, version); - return; - } - - LOGGER.info("Push all temp data tasks success,dataCenter:{} dataInfoId:{} version:{} update!", - dataCenter, dataInfoId, - version); - } else { - LOGGER.warn("Push temp data tasks found error,subscribers version can not be update!"); - } - }); - return pushTaskClosure; - } - private void fireReceivedDataMultiPushTask(Datum datum, List subscriberRegisterIdList, - ScopeEnum scopeEnum, Subscriber subscriber, - PushTaskClosure pushTaskClosure, Map subscriberMap) { + ScopeEnum scopeEnum, Subscriber subscriber, Map subscriberMap) { Collection subscribers = new ArrayList<>(subscriberMap.values()); String dataId = datum.getDataId(); + String clientCell = sessionServerConfig.getClientCell(subscriber.getCell()); Predicate zonePredicate = (zone) -> { - if (!sessionServerConfig.getSessionServerRegion().equals(zone)) { + if (!clientCell.equals(zone)) { if (ScopeEnum.zone == scopeEnum) { // zone scope subscribe only return zone list return true; - } else if (ScopeEnum.dataCenter == scopeEnum) { + } else if (ScopeEnum.dataCenter == scopeEnum || ScopeEnum.global == scopeEnum) { // disable zone config if (sessionServerConfig.isInvalidForeverZone(zone) && !sessionServerConfig.isInvalidIgnored(dataId)) { @@ -194,17 +165,17 @@ private void fireReceivedDataMultiPushTask(Datum datum, List subscriberR } return false; }; + LOGGER.info("Datum push={}",datum); ReceivedData receivedData = ReceivedDataConverter.getReceivedDataMulti(datum, scopeEnum, - subscriberRegisterIdList, sessionServerConfig.getSessionServerRegion(), zonePredicate); + subscriberRegisterIdList, clientCell, zonePredicate); //trigger push to client node Map parameter = new HashMap<>(); parameter.put(receivedData, subscriber.getSourceAddress()); TaskEvent taskEvent = new TaskEvent(parameter, TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); - taskEvent.setTaskClosure(pushTaskClosure); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); - taskLogger.info("send {} taskURL:{},taskScope:{}", taskEvent.getTaskType(), subscriber.getSourceAddress(), - scopeEnum); + taskLogger.info("send {} taskURL:{},taskScope:{},version:{}", taskEvent.getTaskType(), subscriber.getSourceAddress(), + scopeEnum,receivedData.getVersion()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -214,11 +185,9 @@ private Map> getCache(ScopeEnum scope } private void fireUserDataElementPushTask(InetSocketAddress address, Datum datum, - Collection subscribers, - PushTaskClosure pushTaskClosure) { + Collection subscribers) { TaskEvent taskEvent = new TaskEvent(TaskType.USER_DATA_ELEMENT_PUSH_TASK); - taskEvent.setTaskClosure(pushTaskClosure); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); @@ -233,11 +202,9 @@ private void fireUserDataElementPushTask(InetSocketAddress address, Datum datum, } private void fireUserDataElementMultiPushTask(InetSocketAddress address, Datum datum, - Collection subscribers, - PushTaskClosure pushTaskClosure) { + Collection subscribers) { TaskEvent taskEvent = new TaskEvent(TaskType.USER_DATA_ELEMENT_MULTI_PUSH_TASK); - taskEvent.setTaskClosure(pushTaskClosure); taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); @@ -258,6 +225,11 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof DataPushRequest)) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DatumSnapshotTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DatumSnapshotTask.java new file mode 100644 index 000000000..ae7d88f6e --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/DatumSnapshotTask.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler.task; + +import com.alipay.sofa.registry.common.model.DatumSnapshotRequest; +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.task.listener.TaskEvent; + +/** + * + * @author kezhu.wukz + * @version $Id: DatumSnapshotTask.java, v 0.1 2019-06-14 12:15 kezhu.wukz Exp $ + */ +public class DatumSnapshotTask extends AbstractSessionTask { + /** + * transfer data to DataNode + */ + private final DataNodeService dataNodeService; + + private DatumSnapshotRequest datumSnapshotRequest; + + public DatumSnapshotTask(DataNodeService dataNodeService) { + this.dataNodeService = dataNodeService; + } + + @Override + public void execute() { + dataNodeService.sendDatumSnapshot(datumSnapshotRequest); + } + + @Override + public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); + if (obj instanceof DatumSnapshotRequest) { + this.datumSnapshotRequest = (DatumSnapshotRequest) obj; + } else { + throw new IllegalArgumentException("Input task event object error!"); + } + } + + @Override + public String toString() { + return String.format("DATUM_SNAPSHOT_TASK{ taskId=%s, datumSnapshotRequest=%s }", + getTaskId(), datumSnapshotRequest); + } + + @Override + public boolean checkRetryTimes() { + //dataNodeService.sendDatumSnapshot will be retry all the failed + return false; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ProvideDataChangeFetchTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ProvideDataChangeFetchTask.java index 19ffb86c1..2029402f1 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ProvideDataChangeFetchTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ProvideDataChangeFetchTask.java @@ -21,7 +21,6 @@ import com.alipay.sofa.registry.common.model.metaserver.NotifyProvideDataChange; import com.alipay.sofa.registry.common.model.metaserver.ProvideData; import com.alipay.sofa.registry.common.model.store.DataInfo; -import com.alipay.sofa.registry.common.model.store.Subscriber; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.common.model.store.Watcher; import com.alipay.sofa.registry.core.model.ReceivedConfigData; @@ -34,22 +33,17 @@ import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.converter.ReceivedDataConverter; import com.alipay.sofa.registry.server.session.node.service.MetaNodeService; -import com.alipay.sofa.registry.server.session.registry.Registry; -import com.alipay.sofa.registry.server.session.store.Interests; -import com.alipay.sofa.registry.server.session.store.ReSubscribers; +import com.alipay.sofa.registry.server.session.provideData.ProvideDataProcessor; import com.alipay.sofa.registry.server.session.store.Watchers; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.util.CollectionUtils; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; /** * @@ -78,28 +72,30 @@ public class ProvideDataChangeFetchTask extends AbstractSessionTask { private final Exchange boltExchange; - private final Interests sessionInterests; - - private final Registry sessionRegistry; - private NotifyProvideDataChange notifyProvideDataChange; + private ProvideDataProcessor provideDataProcessorManager; + public ProvideDataChangeFetchTask(SessionServerConfig sessionServerConfig, TaskListenerManager taskListenerManager, MetaNodeService metaNodeService, Watchers sessionWatchers, - Exchange boltExchange, Interests sessionInterests, - Registry sessionRegistry) { + Exchange boltExchange, + ProvideDataProcessor provideDataProcessorManager) { this.sessionServerConfig = sessionServerConfig; this.taskListenerManager = taskListenerManager; this.metaNodeService = metaNodeService; this.sessionWatchers = sessionWatchers; this.boltExchange = boltExchange; - this.sessionInterests = sessionInterests; - this.sessionRegistry = sessionRegistry; + this.provideDataProcessorManager = provideDataProcessorManager; } @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof NotifyProvideDataChange)) { @@ -117,42 +113,12 @@ public void execute() { if (notifyProvideDataChange.getDataOperator() != DataOperator.REMOVE) { provideData = metaNodeService.fetchData(dataInfoId); - if (ValueConstants.STOP_PUSH_DATA_SWITCH_DATA_ID.equals(dataInfoId)) { - if (provideData != null) { - if (provideData.getProvideData() == null || provideData.getProvideData().getObject() == null) { - LOGGER.info("Fetch session stop push switch no data existed,config not change!"); - return; - } - String data = (String) provideData.getProvideData().getObject(); - LOGGER.info("Fetch session stop push data switch {} success!", data); - - //receive stop push switch off - if (data != null) { - boolean switchData = Boolean.valueOf(data); - boolean ifChange = sessionServerConfig.isStopPushSwitch() != switchData; - sessionServerConfig.setStopPushSwitch(switchData); - if (!switchData) { - //avoid duplicate false receive - if (ifChange) { - fireReSubscriber(); - } - } else { - //stop push and stop fetch data task - sessionServerConfig.setBeginDataFetchTask(false); - } - } else { - LOGGER.error("Fetch session stop push data switch is null!"); - } - return; - } else { - LOGGER.info("Fetch session stop push switch data null,config not change!"); - } - return; - } if (provideData == null) { LOGGER.warn("Notify provider data Change request {} fetch no provider data!", notifyProvideDataChange); return; } + provideDataProcessorManager.changeDataProcess(provideData); + } DataInfo dataInfo = DataInfo.valueOf(dataInfoId); @@ -161,7 +127,7 @@ public void execute() { for (Channel channel : sessionServer.getChannels()) { //filter all connect client has watcher registerId - String connectId = NetUtil.toAddressString(channel.getRemoteAddress()); + String connectId = NetUtil.toAddressString(channel.getRemoteAddress()) + ValueConstants.CONNECT_ID_SPLIT + NetUtil.toAddressString(channel.getLocalAddress()); Map map = getCache(connectId); List registerIds = new ArrayList<>(); map.forEach((registerId, watchers) -> { @@ -172,11 +138,12 @@ public void execute() { if (!registerIds.isEmpty()) { ReceivedConfigData receivedConfigData; if (notifyProvideDataChange.getDataOperator() == DataOperator.REMOVE) { - receivedConfigData = ReceivedDataConverter.getReceivedConfigData(null, dataInfo, - notifyProvideDataChange.getVersion()); + receivedConfigData = ReceivedDataConverter + .getReceivedConfigData(null, dataInfo, notifyProvideDataChange.getVersion()); } else { - receivedConfigData = ReceivedDataConverter.getReceivedConfigData( - provideData.getProvideData(), dataInfo, provideData.getVersion()); + receivedConfigData = ReceivedDataConverter + .getReceivedConfigData(provideData.getProvideData(), dataInfo, + provideData.getVersion()); } receivedConfigData.setConfiguratorRegistIds(registerIds); firePushTask(receivedConfigData, new URL(channel.getRemoteAddress())); @@ -185,54 +152,6 @@ public void execute() { } } - /** - * open push switch to push all reSubscribers - */ - private void fireReSubscriber() { - - //try catch avoid to error cancel beginDataFetchTask switch on - try { - //begin push fire data fetch task first,avoid reSubscriber push duplicate - sessionRegistry.fetchChangDataProcess(); - } catch (Throwable e) { - LOGGER.error("Open push switch first fetch task execute error",e); - } - - try { - //wait 1 MINUTES for dataFetch task evict duplicate subscriber push - TimeUnit.MINUTES.sleep(1); - } catch (InterruptedException e) { - LOGGER.error("Wait for dataFetch Task Interrupted!"); - } - - //fetch task process 1 minutes,can schedule execute fetch task - sessionServerConfig.setBeginDataFetchTask(true); - - if (this.sessionInterests instanceof ReSubscribers) { - ReSubscribers reSubscriber = (ReSubscribers) sessionInterests; - - Map> reSubscribers = reSubscriber - .getReSubscribers(); - - if (reSubscribers != null && !reSubscribers.isEmpty()) { - reSubscribers.forEach( - (dataInfoId, subscribers) -> fireSubscriberMultiFetchTask(dataInfoId, subscribers.values())); - reSubscriber.clearReSubscribers(); - } - } - } - - private void fireSubscriberMultiFetchTask(String dataInfoId, Collection subscribers) { - //trigger fetch data for subscriber,and push to client node - if (!CollectionUtils.isEmpty(subscribers)) { - TaskEvent taskEvent = new TaskEvent(dataInfoId, TaskType.SUBSCRIBER_MULTI_FETCH_TASK); - taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); - TASK_LOGGER.info("send " + taskEvent.getTaskType() - + " subscribersSize:{},dataInfoId:{}", subscribers.size(), dataInfoId); - taskListenerManager.sendTaskEvent(taskEvent); - } - } - private void firePushTask(ReceivedConfigData receivedConfigData, URL clientUrl) { Map parameter = new HashMap<>(); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PublishDataTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PublishDataTask.java new file mode 100644 index 000000000..b5fa227a3 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PublishDataTask.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler.task; + +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.task.listener.TaskEvent; + +/** + * + * @author kezhu.wukz + * @version $Id: PublishDataTask.java, v 0.1 2019-06-14 12:15 kezhu.wukz Exp $ + */ +public class PublishDataTask extends AbstractSessionTask { + + private final DataNodeService dataNodeService; + + private Publisher publisher; + + public PublishDataTask(DataNodeService dataNodeService) { + this.dataNodeService = dataNodeService; + } + + @Override + public void execute() { + dataNodeService.register(publisher); + } + + @Override + public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); + if (obj instanceof Publisher) { + this.publisher = (Publisher) obj; + } else { + throw new IllegalArgumentException("Input task event object error!"); + } + } + + @Override + public String toString() { + return String + .format("PUBLISH_DATA_TASK{ taskId=%s, publisher=%s }", getTaskId(), publisher); + } + + @Override + public boolean checkRetryTimes() { + //dataNodeService.register will be retry all the failed + return false; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PushTaskClosure.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PushTaskClosure.java index cf7d46ece..9d9910a27 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PushTaskClosure.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/PushTaskClosure.java @@ -18,15 +18,15 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.task.Task; import com.alipay.sofa.registry.task.TaskClosure; import com.alipay.sofa.registry.task.batcher.TaskProcessor.ProcessingResult; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; -import java.util.Map; -import java.util.concurrent.BlockingQueue; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; /** @@ -36,78 +36,74 @@ */ public class PushTaskClosure implements TaskClosure { - private final static Logger LOGGER = LoggerFactory - .getLogger(PushTaskClosure.class); + private final static Logger LOGGER = LoggerFactory + .getLogger(PushTaskClosure.class); - private ConcurrentHashMap taskMap = new ConcurrentHashMap<>(); + private Set tasks = ConcurrentHashMap + .newKeySet(); - private ConcurrentHashMap taskResultMap = new ConcurrentHashMap<>(); + private ConcurrentHashMap taskResultMap = new ConcurrentHashMap<>(); private TaskClosure taskClosure; - private final BlockingQueue completionQueue = new LinkedBlockingQueue<>(); + private final AsyncHashedWheelTimer pushTaskCheckAsyncHashedWheelTimer; - private final ExecutorService pushTaskClosureExecutor; + private final SessionServerConfig sessionServerConfig; - public PushTaskClosure(ExecutorService pushTaskClosureExecutor) { - this.pushTaskClosureExecutor = pushTaskClosureExecutor; + private final String dataInfoId; + + public PushTaskClosure(AsyncHashedWheelTimer pushTaskCheckAsyncHashedWheelTimer, + SessionServerConfig sessionServerConfig, String dataInfoId) { + this.pushTaskCheckAsyncHashedWheelTimer = pushTaskCheckAsyncHashedWheelTimer; + this.sessionServerConfig = sessionServerConfig; + this.dataInfoId = dataInfoId; } @Override public void run(ProcessingResult processingResult, Task task) { if (task != null) { - ProcessingResult existed = taskResultMap - .putIfAbsent(task.getTaskId(), processingResult); - if (existed == null) { - completionQueue.add(task.getTaskId()); + ProcessingResult result = taskResultMap.putIfAbsent(task.getTaskId(), processingResult); + if (result == null) { + if (processingResult == ProcessingResult.Success) { + tasks.remove(task.getTaskId()); + if (tasks.isEmpty()) { + LOGGER.info("Push all tasks success,dataInfoId={}", dataInfoId); + if (taskClosure != null) { + taskClosure.run(ProcessingResult.Success, null); + } + } + } } } } - public void addTask(Task task) { - taskMap.putIfAbsent(task.getTaskId(), task); + public void addTask(TaskEvent taskEvent) { + tasks.add(taskEvent.getTaskId()); } public void start() { - pushTaskClosureExecutor.execute(() -> { - try { - int size = taskMap.size(); - LOGGER.info("Push task queue size {},map size {}", completionQueue.size(), size); - for (int i = 0; i < size; i++) { - String taskId = completionQueue.poll(6000, TimeUnit.MILLISECONDS); - if(taskId != null) { - ProcessingResult result = taskResultMap.get(taskId); - if (result == ProcessingResult.Success) { - taskMap.remove(taskId); - } - } - } - } catch (InterruptedException e) { - LOGGER.error("Push task check InterruptedException!", e); - } + int size = tasks.size(); + LOGGER.info("Push task confirm,dataInfoId={},all task size={}", dataInfoId, size); - if (taskMap.isEmpty()) { - LOGGER.info("Push all tasks success"); - if (taskClosure != null) { - taskClosure.run(ProcessingResult.Success, null); - } + pushTaskCheckAsyncHashedWheelTimer.newTimeout(timeout -> { - } else { - LOGGER.warn("Push tasks found error tasks {} !", taskMap); + if (!tasks.isEmpty()) { + LOGGER.warn("Push tasks found error tasks {},dataInfoId={}!", tasks.size(),dataInfoId); if (taskClosure != null) { taskClosure.run(ProcessingResult.PermanentError, null); } } - }); + + },sessionServerConfig.getPushTaskConfirmWaitTimeout(),TimeUnit.MILLISECONDS); } /** - * Getter method for property taskMap. + * Getter method for property tasks. * - * @return property value of taskMap + * @return property value of tasks */ - public Map getTaskMap() { - return taskMap; + public Set getTasks() { + return tasks; } /** diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedConfigDataPushTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedConfigDataPushTask.java index 168e86474..fdbb11250 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedConfigDataPushTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedConfigDataPushTask.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.Executor; /** * @@ -83,6 +84,11 @@ public void onException(Channel channel, Throwable exception) { receivedConfigData.getDataId(), receivedConfigData.getGroup(), receivedConfigData.getInstanceId(), url); } + + @Override + public Executor getExecutor() { + return null; + } }; clientNodeService.pushWithCallback( @@ -98,6 +104,12 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (obj instanceof Map) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedDataMultiPushTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedDataMultiPushTask.java index dde2fd9be..2752e70f7 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedDataMultiPushTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/ReceivedDataMultiPushTask.java @@ -16,9 +16,18 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + import com.alipay.sofa.registry.common.model.PushDataRetryRequest; +import com.alipay.sofa.registry.common.model.store.DataInfo; import com.alipay.sofa.registry.common.model.store.Subscriber; import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.core.model.DataBox; import com.alipay.sofa.registry.core.model.ReceivedData; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; @@ -29,24 +38,19 @@ import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.service.ClientNodeService; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.strategy.ReceivedDataMultiPushTaskStrategy; -import com.alipay.sofa.registry.task.Task; import com.alipay.sofa.registry.task.TaskClosure; import com.alipay.sofa.registry.task.batcher.TaskProcessor.ProcessingResult; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.timer.AsyncHashedWheelTimer; -import java.util.Collection; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; - /** * * @author shangyu.wh * @version $Id: SubscriberRegisterPushTask.java, v 0.1 2017-12-11 20:57 shangyu.wh Exp $ */ -public class ReceivedDataMultiPushTask extends AbstractSessionTask implements TaskClosure { +public class ReceivedDataMultiPushTask extends AbstractSessionTask { private static final Logger LOGGER = LoggerFactory.getLogger("SESSION-PUSH", "[Receive]"); @@ -61,68 +65,89 @@ public class ReceivedDataMultiPushTask extends AbstractSessionTask implements Ta private Collection subscribers; private ReceivedDataMultiPushTaskStrategy receivedDataMultiPushTaskStrategy; private AsyncHashedWheelTimer asyncHashedWheelTimer; + private Interests sessionInterests; + + private String dataPush; public ReceivedDataMultiPushTask(SessionServerConfig sessionServerConfig, ClientNodeService clientNodeService, ExecutorManager executorManager, Exchange boltExchange, ReceivedDataMultiPushTaskStrategy receivedDataMultiPushTaskStrategy, - AsyncHashedWheelTimer asyncHashedWheelTimer) { + AsyncHashedWheelTimer asyncHashedWheelTimer, + Interests sessionInterests) { this.sessionServerConfig = sessionServerConfig; this.clientNodeService = clientNodeService; this.executorManager = executorManager; this.boltExchange = boltExchange; this.receivedDataMultiPushTaskStrategy = receivedDataMultiPushTaskStrategy; this.asyncHashedWheelTimer = asyncHashedWheelTimer; + this.sessionInterests = sessionInterests; } @Override public void execute() { + Object receivedDataPush = null; + try { + if (sessionServerConfig.isStopPushSwitch()) { + LOGGER + .info( + "Stop Push ReceivedData with switch on! dataId: {},group: {},Instance: {}, url: {}", + receivedData.getDataId(), receivedData.getGroup(), + receivedData.getInstanceId(), url); + return; + } - if (sessionServerConfig.isStopPushSwitch()) { - LOGGER - .info( - "Stop Push ReceivedData with switch on! dataId: {},group: {},Instance: {}, url: {}", - receivedData.getDataId(), receivedData.getGroup(), - receivedData.getInstanceId(), url); - return; - } - - Object receivedDataPush = receivedDataMultiPushTaskStrategy.convert2PushData(receivedData, - url); - - CallbackHandler callbackHandler = new CallbackHandler() { - @Override - public void onCallback(Channel channel, Object message) { - LOGGER.info( - "Push ReceivedData success! dataId:{},group:{},Instance:{},version:{},url: {}", - receivedData.getDataId(), receivedData.getGroup(), - receivedData.getInstanceId(), receivedData.getVersion(), url); - - if (taskClosure != null) { - confirmCallBack(true); + receivedDataPush = receivedDataMultiPushTaskStrategy + .convert2PushData(receivedData, url); + + final Object finalReceivedDataPush = receivedDataPush; + CallbackHandler callbackHandler = new CallbackHandler() { + @Override + public void onCallback(Channel channel, Object message) { + + if (taskClosure != null) { + confirmCallBack(true); + } + LOGGER + .info( + "Push ReceivedData success! dataId:{},group:{},Instance:{},version:{},url: {},dataPush:{}", + receivedData.getDataId(), receivedData.getGroup(), + receivedData.getInstanceId(), receivedData.getVersion(), url, dataPush); } - } - @Override - public void onException(Channel channel, Throwable exception) { - LOGGER.error( - "Push ReceivedData error! dataId:{},group:{},Instance:{},version:{},url: {}", - receivedData.getDataId(), receivedData.getGroup(), - receivedData.getInstanceId(), receivedData.getVersion(), url, exception); + @Override + public void onException(Channel channel, Throwable exception) { + try { + LOGGER + .error( + "Push ReceivedData error! dataId:{},group:{},Instance:{},version:{},url: {},dataPush:{}", + receivedData.getDataId(), receivedData.getGroup(), + receivedData.getInstanceId(), receivedData.getVersion(), url, + dataPush, exception); + + if (taskClosure != null) { + throw new RuntimeException( + "Push ReceivedData got exception from callback!"); + } else { + retrySendReceiveData(new PushDataRetryRequest(finalReceivedDataPush, + url)); + } + } finally { + if (taskClosure != null) { + confirmCallBack(false); + } + } + } - if (taskClosure != null) { - confirmCallBack(false); - throw new RuntimeException("Push ReceivedData got exception from callback!"); - } else { - retrySendReceiveData(new PushDataRetryRequest(receivedDataPush, url)); + @Override + public Executor getExecutor() { + return null; } - } - }; + }; - try { clientNodeService.pushWithCallback(receivedDataPush, url, callbackHandler); - } catch (Exception e) { + } catch (Throwable e) { if (taskClosure != null) { confirmCallBack(false); throw e; @@ -149,36 +174,62 @@ private void retrySendReceiveData(PushDataRetryRequest pushDataRetryRequest) { if (channel != null && channel.isConnected()) { - asyncHashedWheelTimer.newTimeout(timeout -> { + asyncHashedWheelTimer.newTimeout(timeout -> { try { clientNodeService.pushWithCallback(infoPackage, targetUrl, new CallbackHandler() { @Override public void onCallback(Channel channel, Object message) { - LOGGER.info("Retry Push ReceivedData success! dataId:{}, group:{},url:{},retryTimes:{}", - receivedData.getDataId(), receivedData.getGroup(), targetUrl,retryTimes); + LOGGER.info( + "Retry Push ReceivedData success! dataId:{}, group:{},url:{},taskId:{},dataPush:{},retryTimes:{}", + receivedData.getDataId(), receivedData.getGroup(), targetUrl, getTaskId(), + dataPush, retryTimes); } @Override public void onException(Channel channel, Throwable exception) { - LOGGER.error("Retry Push ReceivedData callback error! url:{}, dataId:{}, group:{},taskId:{},retryTimes:{}", targetUrl, - receivedData.getDataId(), receivedData.getGroup(),getTaskId(),retryTimes); + LOGGER.error( + "Retry Push ReceivedData callback error! url:{}, dataId:{}, group:{},taskId:{},dataPush:{},retryTimes:{}", + targetUrl, receivedData.getDataId(), receivedData.getGroup(), getTaskId(), + dataPush, retryTimes); retrySendReceiveData(pushDataRetryRequest); } + + @Override + public Executor getExecutor() { + return null; + } }); } catch (Exception e) { - LOGGER.error("Retry Push ReceivedData error! url:{}, dataId:{}, group:{},taskId:{},retryTimes:{}", targetUrl, - receivedData.getDataId(), receivedData.getGroup(),getTaskId(),retryTimes); + LOGGER.error( + "Retry Push ReceivedData error! url:{}, dataId:{}, group:{},taskId:{},dataPush:{},retryTimes:{}", + targetUrl, receivedData.getDataId(), receivedData.getGroup(), getTaskId(), dataPush, + retryTimes); retrySendReceiveData(pushDataRetryRequest); } },getBlockTime(retryTimes),TimeUnit.MILLISECONDS); } else { - LOGGER.error("Retry Push ReceivedData error, connect be null or disconnected,stop retry!dataId:{}, group:{},url:{},taskId:{},retryTimes:{}", - receivedData.getDataId(), receivedData.getGroup(), targetUrl,getTaskId(),retryTimes); + LOGGER.error( + "Retry Push ReceivedData error, connect be null or disconnected,stop retry!dataId:{}, group:{},url:{},taskId:{},dataPush:{},retryTimes:{}", + receivedData.getDataId(), receivedData.getGroup(), targetUrl, getTaskId(), dataPush, + retryTimes); } } else { - LOGGER.error("Retry Push ReceivedData times have exceeded!dataId:{}, group:{},url:{},taskId:{},retryTimes:{}", - receivedData.getDataId(), receivedData.getGroup(), targetUrl,getTaskId(),retryTimes); + //set sessionInterests dataInfoId version zero + DataInfo dataInfo = new DataInfo(receivedData.getInstanceId(), receivedData.getDataId(), + receivedData.getGroup()); + boolean result = sessionInterests.checkAndUpdateInterestVersionZero(receivedData.getSegment(), dataInfo.getDataInfoId()); + if (result) { + LOGGER.warn("Retry Push ReceivedData times have exceeded,set sessionInterests dataInfoId version zero! url:{},taskId:{},dataPush:{},retryTimes:{},dataCenter:{},dataInfoId:{}!", + targetUrl,getTaskId(),dataPush,retryTimes, + receivedData.getSegment(), + dataInfo.getDataInfoId()); + } else { + LOGGER.warn("Retry Push ReceivedData times have exceeded,but set sessionInterests dataInfoId version zero fail!url:{},taskId:{},dataPush:{},retryTimes:{},dataCenter:{},dataInfoId:{}!", + targetUrl,getTaskId(),dataPush,retryTimes, + receivedData.getSegment(), + dataInfo.getDataInfoId()); + } } } } @@ -191,6 +242,11 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (obj instanceof Map) { @@ -210,12 +266,24 @@ public void setTaskEvent(TaskEvent taskEvent) { } } - taskClosure = taskEvent.getTaskClosure(); + if (receivedData != null && receivedData.getData() != null) { + StringBuilder sb = new StringBuilder(); + sb.append("["); + Map> map = receivedData.getData(); + if (!map.isEmpty()) { - if (taskClosure instanceof PushTaskClosure) { - ((PushTaskClosure) taskClosure).addTask(this); + for (Map.Entry> entry1 : map.entrySet()) { + sb.append(entry1.getKey()).append("="); + int size = entry1.getValue() != null ? entry1.getValue().size() : 0; + sb.append(size).append(","); + } + } + sb.append("]"); + dataPush = sb.toString(); } + taskClosure = taskEvent.getTaskClosure(); + subscribers = (Collection) taskEvent .getAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS); //taskClosure must confirm all subscriber push success @@ -275,10 +343,4 @@ public boolean checkRetryTimes() { return checkRetryTimes(sessionServerConfig.getReceivedDataMultiPushTaskRetryTimes()); } - @Override - public void run(ProcessingResult processingResult, Task task) { - if (taskClosure != null) { - taskClosure.run(processingResult, task); - } - } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/RenewDatumTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/RenewDatumTask.java new file mode 100644 index 000000000..04ee1814a --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/RenewDatumTask.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler.task; + +import com.alipay.sofa.registry.common.model.RenewDatumRequest; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.server.session.registry.SessionRegistry; +import com.alipay.sofa.registry.task.listener.TaskEvent; + +/** + * + * @author kezhu.wukz + * @version $Id: RenewDatumTask.java, v 0.1 2019-06-14 12:15 kezhu.wukz Exp $ + */ +public class RenewDatumTask extends AbstractSessionTask { + + private static final Logger RENEW_LOGGER = LoggerFactory.getLogger( + ValueConstants.LOGGER_NAME_RENEW, + "[RenewDatumTask]"); + + private final DataNodeService dataNodeService; + + private final SessionServerConfig sessionServerConfig; + + private final SessionRegistry sessionRegistry; + + private RenewDatumRequest renewDatumRequest; + + public RenewDatumTask(SessionServerConfig sessionServerConfig, DataNodeService dataNodeService, + SessionRegistry sessionRegistry) { + this.sessionServerConfig = sessionServerConfig; + this.dataNodeService = dataNodeService; + this.sessionRegistry = sessionRegistry; + } + + @Override + public void execute() { + + try { + Boolean result = dataNodeService.renewDatum(renewDatumRequest); + if (!result) { + RENEW_LOGGER.info( + "Renew datum request to dataNode got digest different! renewDatumRequest={}", + renewDatumRequest); + + // send snapshot datum for the corresponding connId + sessionRegistry.sendDatumSnapshot(renewDatumRequest.getConnectId(), + renewDatumRequest.getDataServerIP()); + + } + } catch (Exception e) { + RENEW_LOGGER.error(String.format( + "Renew datum request to dataNode error! renewDatumRequest=%s, errorMsg=%s", + renewDatumRequest, e.getMessage()), e); + } + } + + @Override + public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); + if (obj instanceof RenewDatumRequest) { + this.renewDatumRequest = (RenewDatumRequest) obj; + } else { + throw new IllegalArgumentException("Input task event object error!"); + } + } + + @Override + public String toString() { + return String.format("RENEW_DATUM_TASK{ taskId=%s, renewDatumRequest=%s }", getTaskId(), + renewDatumRequest); + } + + @Override + public boolean checkRetryTimes() { + return checkRetryTimes(sessionServerConfig.getRenewDatumTaskRetryTimes()); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SessionRegisterDataTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SessionRegisterDataTask.java index 0640760ed..92b300455 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SessionRegisterDataTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SessionRegisterDataTask.java @@ -16,26 +16,28 @@ */ package com.alipay.sofa.registry.server.session.scheduler.task; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.dataserver.SessionServerRegisterRequest; -import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.net.NetUtil; import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.Client; import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.bolt.BoltChannel; import com.alipay.sofa.registry.remoting.exchange.Exchange; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.node.SessionProcessIdGenerator; -import com.alipay.sofa.registry.server.session.node.service.DataNodeService; import com.alipay.sofa.registry.task.listener.TaskEvent; -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; - /** * * @author shangyu.wh + * @author kezhu.wukz * @version $Id: SessionRegisterDataTask.java, v 0.1 2018-04-16 16:07 shangyu.wh Exp $ */ public class SessionRegisterDataTask extends AbstractSessionTask { @@ -44,16 +46,13 @@ public class SessionRegisterDataTask extends AbstractSessionTask { SessionRegisterDataTask.class, "[Task]"); private final Exchange boltExchange; - private final DataNodeService dataNodeService; private final SessionServerConfig sessionServerConfig; private SessionServerRegisterRequest sessionServerRegisterRequest; - private URL dataUrl; + private BoltChannel channel; - public SessionRegisterDataTask(Exchange boltExchange, DataNodeService dataNodeService, - SessionServerConfig sessionServerConfig) { + public SessionRegisterDataTask(Exchange boltExchange, SessionServerConfig sessionServerConfig) { this.boltExchange = boltExchange; - this.dataNodeService = dataNodeService; this.sessionServerConfig = sessionServerConfig; } @@ -64,11 +63,16 @@ public boolean checkRetryTimes() { @Override public void setTaskEvent(TaskEvent taskEvent) { - Object obj = taskEvent.getEventObj(); - if (obj instanceof URL) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); - this.dataUrl = (URL) obj; + if (obj instanceof BoltChannel) { + this.channel = (BoltChannel) obj; } else { throw new IllegalArgumentException("Input task event object error!"); @@ -78,11 +82,12 @@ public void setTaskEvent(TaskEvent taskEvent) { if (sessionServer != null) { Collection chs = sessionServer.getChannels(); - Set clientHosts = new HashSet<>(); - chs.forEach(channel -> clientHosts.add(NetUtil.toAddressString(channel.getRemoteAddress()))); + Set connectIds = new HashSet<>(); + chs.forEach(channel -> connectIds.add(NetUtil.toAddressString(channel.getRemoteAddress()) + + ValueConstants.CONNECT_ID_SPLIT + NetUtil.toAddressString(channel.getLocalAddress()))); sessionServerRegisterRequest = new SessionServerRegisterRequest( - SessionProcessIdGenerator.getSessionProcessId(), clientHosts); + SessionProcessIdGenerator.getSessionProcessId(), connectIds); } else { LOGGER.error("get null session server,please check server started before register!port {}", sessionServerConfig.getServerPort()); @@ -93,14 +98,39 @@ public void setTaskEvent(TaskEvent taskEvent) { @Override public void execute() { - dataNodeService.registerSessionProcessId(sessionServerRegisterRequest, dataUrl); + if (!channel.isConnected()) { + return; + } + + Client sessionClient = boltExchange.getClient(Exchange.DATA_SERVER_TYPE); + try { + sessionClient.sendSync(channel, sessionServerRegisterRequest, + sessionServerConfig.getDataNodeExchangeTimeOut()); + } catch (Exception e) { + if (isLastRetry()) { + LOGGER + .error( + "Register to DataServer({}/{}:{}) error for multiple times, so close this channel (let bolt reconnect it)", + channel.getLocalAddress(), channel.getRemoteAddress().getHostString(), + channel.getRemoteAddress().getPort()); + channel.close(); + } + throw e; + } + + } + + protected boolean isLastRetry() { + return getExecCount() >= sessionServerConfig.getSessionRegisterDataServerTaskRetryTimes(); } @Override public String toString() { return "SESSION_REGISTER_DATA_TASK{" + "taskId='" + taskId + '\'' + ", sessionServerRegisterRequest=" + sessionServerRegisterRequest.getProcessId() - + ", clientList=" + sessionServerRegisterRequest.getClientHosts().size() - + ", dataUrl=" + dataUrl + '}'; + + ", clientList=" + sessionServerRegisterRequest.getConnectIds().size() + + ", channel=" + channel.getLocalAddress() + "/" + + channel.getRemoteAddress().getHostString() + ':' + + channel.getRemoteAddress().getPort() + '}'; } } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberMultiFetchTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberMultiFetchTask.java index ffb1495c0..b2b35d1e3 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberMultiFetchTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberMultiFetchTask.java @@ -63,6 +63,11 @@ public SubscriberMultiFetchTask(SessionServerConfig sessionServerConfig, @Override public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof String)) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberPushEmptyTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberPushEmptyTask.java new file mode 100644 index 000000000..027b2d7b4 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberPushEmptyTask.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler.task; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.BaseInfo.ClientVersion; +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.core.model.ReceivedData; +import com.alipay.sofa.registry.core.model.ScopeEnum; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.converter.ReceivedDataConverter; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskEvent.TaskType; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import com.alipay.sofa.registry.util.DatumVersionUtil; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * @author shangyu.wh + * @version $Id: SubscriberRegisterFetchTask.java, v 0.1 2017-12-07 16:23 shangyu.wh Exp $ + */ +public class SubscriberPushEmptyTask extends AbstractSessionTask { + + private static final Logger taskLogger = LoggerFactory.getLogger( + SubscriberPushEmptyTask.class, "[Task]"); + + protected final SessionServerConfig sessionServerConfig; + /** + * trigger task com.alipay.sofa.registry.server.meta.listener process + */ + protected final TaskListenerManager taskListenerManager; + protected Subscriber subscriber; + + public SubscriberPushEmptyTask(SessionServerConfig sessionServerConfig, + TaskListenerManager taskListenerManager) { + this.sessionServerConfig = sessionServerConfig; + this.taskListenerManager = taskListenerManager; + } + + @Override + public long getExpiryTime() { + return -1; + } + + @Override + public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); + + if (!(obj instanceof Subscriber)) { + throw new IllegalArgumentException("Input task event object error!"); + } + + this.subscriber = (Subscriber) obj; + } + + @Override + public void execute() { + executeTask(); + } + + protected void executeTask() { + + if (subscriber == null) { + throw new IllegalArgumentException("Subscriber can not be null!"); + } + + List subscriberRegisterIdList = Collections.singletonList(subscriber + .getRegisterId()); + + boolean isOldVersion = !ClientVersion.StoreData.equals(subscriber.getClientVersion()); + + switch (subscriber.getScope()) { + case zone: + if (isOldVersion) { + fireUserDataElementPushTask(); + } else { + fireReceivedDataPushTask(subscriberRegisterIdList, ScopeEnum.zone); + } + break; + + case dataCenter: + if (isOldVersion) { + fireUserDataElementMultiPushTask(); + } else { + fireReceivedDataPushTask(subscriberRegisterIdList, ScopeEnum.dataCenter); + } + break; + + case global: + + fireReceivedDataPushTask(subscriberRegisterIdList, ScopeEnum.global); + + break; + default: + break; + } + + } + + protected void fireReceivedDataPushTask(List subscriberRegisterIdList, + ScopeEnum scopeEnum) { + ReceivedData receivedData = ReceivedDataConverter.getReceivedDataMulti( + subscriber.getDataId(), subscriber.getGroup(), subscriber.getInstanceId(), + ValueConstants.DEFAULT_DATA_CENTER, scopeEnum, subscriberRegisterIdList, + subscriber.getCell()); + //no datum set version current timestamp + receivedData.setVersion(DatumVersionUtil.nextId()); + firePush(receivedData); + } + + protected void firePush(ReceivedData receivedData) { + //trigger push to client node + Map parameter = new HashMap<>(); + parameter.put(receivedData, subscriber.getSourceAddress()); + TaskEvent taskEvent = new TaskEvent(parameter, TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); + taskLogger.info("send {} taskURL:{},taskScope", taskEvent.getTaskType(), + subscriber.getSourceAddress(), receivedData.getScope()); + taskListenerManager.sendTaskEvent(taskEvent); + } + + protected void fireUserDataElementPushTask() { + + //no datum + Datum datum = new Datum(); + datum.setDataId(subscriber.getDataId()); + datum.setInstanceId(subscriber.getInstanceId()); + datum.setGroup(subscriber.getGroup()); + //no datum set version current timestamp + datum.setVersion(DatumVersionUtil.nextId()); + datum.setPubMap(new HashMap<>()); + datum.setDataCenter(ValueConstants.DEFAULT_DATA_CENTER); + + Collection subscribers = new ArrayList<>(); + subscribers.add(subscriber); + + TaskEvent taskEvent = new TaskEvent(subscriber, TaskType.USER_DATA_ELEMENT_PUSH_TASK); + + taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); + taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); + taskEvent.setAttribute(Constant.PUSH_CLIENT_URL, subscriber.getSourceAddress()); + + taskLogger.info("send {} taskURL:{}", taskEvent.getTaskType(), + subscriber.getSourceAddress()); + taskListenerManager.sendTaskEvent(taskEvent); + } + + protected void fireUserDataElementMultiPushTask() { + + //no datum + Datum datum = new Datum(); + datum.setDataId(subscriber.getDataId()); + datum.setInstanceId(subscriber.getInstanceId()); + datum.setGroup(subscriber.getGroup()); + //no datum set version as mini as + datum.setVersion(DatumVersionUtil.nextId()); + datum.setPubMap(new HashMap<>()); + datum.setDataCenter(ValueConstants.DEFAULT_DATA_CENTER); + + Collection subscribers = new ArrayList<>(); + subscribers.add(subscriber); + + TaskEvent taskEvent = new TaskEvent(subscriber, TaskType.USER_DATA_ELEMENT_MULTI_PUSH_TASK); + taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, subscribers); + taskEvent.setAttribute(Constant.PUSH_CLIENT_DATUM, datum); + taskEvent.setAttribute(Constant.PUSH_CLIENT_URL, subscriber.getSourceAddress()); + + taskLogger.info("send {} taskURL:{}", taskEvent.getTaskType(), + subscriber.getSourceAddress()); + taskListenerManager.sendTaskEvent(taskEvent); + } + + @Override + public boolean checkRetryTimes() { + return checkRetryTimes(sessionServerConfig.getSubscriberRegisterFetchRetryTimes()); + } + + @Override + public String toString() { + return "SubscriberPushEmptyTask{" + "taskId='" + getTaskId() + '\'' + ", subscriber=" + + subscriber + ", expiryTime='" + getExpiryTime() + '\'' + '}'; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberRegisterFetchTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberRegisterFetchTask.java index 815325337..9a539dd81 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberRegisterFetchTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/SubscriberRegisterFetchTask.java @@ -66,6 +66,12 @@ public long getExpiryTime() { @Override public void setTaskEvent(TaskEvent taskEvent) { + + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof Subscriber)) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/UnPublishDataTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/UnPublishDataTask.java new file mode 100644 index 000000000..46fa86298 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/UnPublishDataTask.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.scheduler.task; + +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.server.session.node.service.DataNodeService; +import com.alipay.sofa.registry.task.listener.TaskEvent; + +/** + * + * @author kezhu.wukz + * @version $Id: UnPublishDataTask.java, v 0.1 2019-06-14 12:15 kezhu.wukz Exp $ + */ +public class UnPublishDataTask extends AbstractSessionTask { + + private final DataNodeService dataNodeService; + + private Publisher unPublisher; + + public UnPublishDataTask(DataNodeService dataNodeService) { + this.dataNodeService = dataNodeService; + } + + @Override + public void execute() { + dataNodeService.unregister(unPublisher); + } + + @Override + public void setTaskEvent(TaskEvent taskEvent) { + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + + Object obj = taskEvent.getEventObj(); + if (obj instanceof Publisher) { + this.unPublisher = (Publisher) obj; + } else { + throw new IllegalArgumentException("Input task event object error!"); + } + } + + @Override + public String toString() { + return String.format("UN_PUBLISH_DATA_TASK{ taskId=%s, unPublisher=%s }", getTaskId(), + unPublisher); + } + + @Override + public boolean checkRetryTimes() { + //dataNodeService.unregister will be retry all the failed + return false; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/WatcherRegisterFetchTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/WatcherRegisterFetchTask.java index ad5bdf1b0..441b9dd07 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/WatcherRegisterFetchTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/task/WatcherRegisterFetchTask.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; /** * @@ -58,6 +59,8 @@ public class WatcherRegisterFetchTask extends AbstractSessionTask { private Watcher watcher; + private static final int TRY_COUNT = 3; + public WatcherRegisterFetchTask(SessionServerConfig sessionServerConfig, TaskListenerManager taskListenerManager, MetaNodeService metaNodeService) { @@ -68,6 +71,12 @@ public WatcherRegisterFetchTask(SessionServerConfig sessionServerConfig, @Override public void setTaskEvent(TaskEvent taskEvent) { + + //taskId create from event + if (taskEvent.getTaskId() != null) { + setTaskId(taskEvent.getTaskId()); + } + Object obj = taskEvent.getEventObj(); if (!(obj instanceof Watcher)) { @@ -88,17 +97,30 @@ public void execute() { boolean isOldVersion = !ClientVersion.StoreData.equals(watcher.getClientVersion()); - ProvideData provideData = metaNodeService.fetchData(watcher.getDataInfoId()); - if (provideData != null) { - if (!isOldVersion) { - DataInfo dataInfo = DataInfo.valueOf(provideData.getDataInfoId()); - ReceivedConfigData receivedConfigData = ReceivedDataConverter - .getReceivedConfigData(provideData.getProvideData(), dataInfo, - provideData.getVersion()); - receivedConfigData.setConfiguratorRegistIds(subscriberRegisterIdList); - firePushTask(receivedConfigData); + ProvideData provideData = null; + + for (int tryCount = 0; tryCount < TRY_COUNT; tryCount++) { + try { + provideData = metaNodeService.fetchData(watcher.getDataInfoId()); + break; + } catch (Exception e) { + randomDelay(3000); } } + + if (provideData == null) { + taskLogger.error("Fetch provider data error,set null value return.dataInfoId={}", + watcher.getDataId()); + provideData = new ProvideData(null, watcher.getDataInfoId(), null); + } + + if (!isOldVersion) { + DataInfo dataInfo = DataInfo.valueOf(provideData.getDataInfoId()); + ReceivedConfigData receivedConfigData = ReceivedDataConverter.getReceivedConfigData( + provideData.getProvideData(), dataInfo, provideData.getVersion()); + receivedConfigData.setConfiguratorRegistIds(subscriberRegisterIdList); + firePushTask(receivedConfigData); + } } private void firePushTask(ReceivedConfigData receivedConfigData) { @@ -114,6 +136,16 @@ public boolean checkRetryTimes() { return checkRetryTimes(sessionServerConfig.getSubscriberRegisterFetchRetryTimes()); } + private void randomDelay(int max) { + Random random = new Random(); + int randomNum = random.nextInt(max); + try { + Thread.sleep(randomNum); + } catch (InterruptedException e) { + taskLogger.error("[TimeUtil] random delay error", e); + } + } + @Override public String toString() { return "WATCHER_REGISTER_FETCH_TASK{" + "taskId='" + getTaskId() + '\'' + ", watcher=" diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/timertask/SyncClientsHeartbeatTask.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/timertask/SyncClientsHeartbeatTask.java index 7f2d58464..eaf14b8a0 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/timertask/SyncClientsHeartbeatTask.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/scheduler/timertask/SyncClientsHeartbeatTask.java @@ -16,9 +16,23 @@ */ package com.alipay.sofa.registry.server.session.scheduler.timertask; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ThreadPoolExecutor; + +import com.alipay.remoting.ProtocolCode; +import com.alipay.remoting.ProtocolManager; +import com.alipay.remoting.rpc.protocol.RpcProtocol; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; + import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.metrics.TaskMetrics; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.listener.ReceivedDataMultiPushTaskListener; import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; import com.alipay.sofa.registry.server.session.store.DataStore; @@ -30,13 +44,6 @@ import com.alipay.sofa.registry.task.listener.TaskListener; import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; - -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ThreadPoolExecutor; /** * The type Sync clients heartbeat task. @@ -54,8 +61,13 @@ public class SyncClientsHeartbeatTask { "SESSION-PROFILE-DIGEST", "[ExecutorMetrics]"); - public static final String SYMBOLIC1 = " ├─ "; - public static final String SYMBOLIC2 = " └─ "; + public static final String SYMBOLIC = " └─ "; + + @Autowired + private Exchange boltExchange; + + @Autowired + private SessionServerConfig sessionServerConfig; /** * store subscribers @@ -75,11 +87,17 @@ public class SyncClientsHeartbeatTask { @Autowired private DataStore sessionDataStore; - @Autowired - private ExecutorManager executorManager; - @Autowired private TaskListener receivedDataMultiPushTaskListener; + private final TaskMetrics taskMetrics = TaskMetrics.getInstance(); + + public SyncClientsHeartbeatTask() { + + ThreadPoolExecutor boltDefaultExecutor = (ThreadPoolExecutor) ProtocolManager + .getProtocol(ProtocolCode.fromBytes(RpcProtocol.PROTOCOL_CODE)).getCommandHandler() + .getDefaultExecutor(); + taskMetrics.registerThreadExecutor("Session-BoltDefaultExecutor", boltDefaultExecutor); + } @Scheduled(initialDelayString = "${session.server.syncHeartbeat.fixedDelay}", fixedDelayString = "${session.server.syncHeartbeat.fixedDelay}") public void syncCounte() { @@ -87,8 +105,15 @@ public void syncCounte() { long countPub = sessionDataStore.count(); long countSubW = sessionWatchers.count(); - CONSOLE_COUNT_LOGGER.info("Subscriber count: {}, Publisher count: {}, Watcher count: {},", - countSub, countPub, countSubW); + int channelCount = 0; + Server sessionServer = boltExchange.getServer(sessionServerConfig.getServerPort()); + if (sessionServer != null) { + channelCount = sessionServer.getChannelCount(); + } + + CONSOLE_COUNT_LOGGER.info( + "Subscriber count: {}, Publisher count: {}, Watcher count: {}, Connection count: {}", + countSub, countPub, countSubW, channelCount); } @Scheduled(initialDelayString = "${session.server.printTask.fixedDelay}", fixedDelayString = "${session.server.printTask.fixedDelay}") @@ -112,11 +137,7 @@ protected void logInfo(StringBuilder sb0, Map taskDispat .hasNext();) { Entry entry = i.next(); AcceptorExecutor acceptorExecutor = entry.getValue().getAcceptorExecutor(); - String outterTreeSymbol = SYMBOLIC1; - if (!i.hasNext()) { - outterTreeSymbol = SYMBOLIC2; - } - sb.append(outterTreeSymbol).append(entry.getKey()); + sb.append(SYMBOLIC).append(entry.getKey()); sb.append(", AcceptedTasks:").append(acceptorExecutor.getAcceptedTasks()); sb.append(", ReplayedTasks:").append(acceptorExecutor.getReplayedTasks()); sb.append(", QueueOverflows:").append(acceptorExecutor.getQueueOverflows()); @@ -130,39 +151,7 @@ protected void logInfo(StringBuilder sb0, Map taskDispat @Scheduled(initialDelayString = "${session.server.printTask.fixedDelay}", fixedDelayString = "${session.server.printTask.fixedDelay}") public void printExecutorTaskExecute() { - - Map reportExecutors = executorManager.getReportExecutors(); - if (reportExecutors != null) { - - StringBuilder sb = new StringBuilder(); - logInfoExecutor(sb, reportExecutors, "ExecutorMetrics"); - EXE_LOGGER.info(sb.toString()); - } - - } - - protected void logInfoExecutor(StringBuilder sb0, Map reportExecutors, String info) { - sb0.append("\n").append(info).append(" >>>>>>>"); - StringBuilder sb = new StringBuilder(); - for (Iterator> i = reportExecutors.entrySet().iterator(); i.hasNext(); ) { - Entry entry = i.next(); - String executorName = entry.getKey(); - - MetricRegistry metricRegistry = TaskMetrics.getInstance().getMetricRegistry(); - Map map = metricRegistry.getGauges((name, value) -> name.startsWith(executorName)); - - String outterTreeSymbol = SYMBOLIC1; - if (!i.hasNext()) { - outterTreeSymbol = SYMBOLIC2; - } - sb.append(outterTreeSymbol).append(executorName); - map.forEach((key, gauge) -> { - String name = key.substring(executorName.length() + 1); - sb.append(", ").append(name).append(":").append(gauge.getValue()); - }); - sb.append("\n"); - } - sb0.append("\n").append(sb); + EXE_LOGGER.info(TaskMetrics.getInstance().metricsString()); } @Scheduled(initialDelayString = "${session.server.printTask.fixedDelay}", fixedDelayString = "${session.server.printTask.fixedDelay}") diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/DataStore.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/DataStore.java index 2b1fe2970..455057c3c 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/DataStore.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/DataStore.java @@ -19,6 +19,7 @@ import com.alipay.sofa.registry.common.model.store.Publisher; import java.util.Collection; +import java.util.Map; /** * @@ -27,8 +28,32 @@ */ public interface DataStore extends DataManager { + /** + * get all publishers by dataInfoId + * @param dataInfoId + * @return + */ Collection getStoreDataByDataInfoId(String dataInfoId); + /*** + * get Publiser by registerId and dataInfoId + * @param registerId + * @param dataInfoId + * @return + */ Publisher queryById(String registerId, String dataInfoId); + /** + * get all publisher dataInfoIds + * + * @return + */ + Collection getStoreDataInfoIds(); + + /** + * get all publishers group by connectId + * @return + */ + Map> getConnectPublishers(); + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Interests.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Interests.java index 3fcf99a39..6206f8851 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Interests.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Interests.java @@ -16,13 +16,14 @@ */ package com.alipay.sofa.registry.server.session.store; -import com.alipay.sofa.registry.common.model.store.Subscriber; -import com.alipay.sofa.registry.core.model.ScopeEnum; - import java.net.InetSocketAddress; import java.util.Collection; +import java.util.List; import java.util.Map; +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.core.model.ScopeEnum; + /** * * @author shangyu.wh @@ -62,6 +63,14 @@ public interface Interests extends DataManager { */ boolean checkAndUpdateInterestVersions(String dataCenter, String dataInfoId, Long version); + /** + * set subscribers interest dataInfoId version zero + * @param dataCenter + * @param dataInfoId + * @return + */ + boolean checkAndUpdateInterestVersionZero(String dataCenter, String dataInfoId); + /** * get all subscriber dataInfoIds * @@ -70,11 +79,27 @@ public interface Interests extends DataManager { Collection getInterestDataInfoIds(); /** - * get subscribers whith specify dataInfo and scope,and order by source InetSocketAddress + * get subscribers whith specify dataInfo and scope,and group by source InetSocketAddress * @param dataInfoId * @param scope * @return */ Map> querySubscriberIndex(String dataInfoId, ScopeEnum scope); + + /** + * get subscriber by dataInfoId and registerId + * @param registerId + * @param dataInfoId + * @return + */ + Subscriber queryById(String registerId, String dataInfoId); + + /** + * get all subscribers group by connectId + * @return + */ + Map> getConnectSubscribers(); + + List getDataCenters(); } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionDataStore.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionDataStore.java index 080e46bd8..9fba16f88 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionDataStore.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionDataStore.java @@ -16,10 +16,6 @@ */ package com.alipay.sofa.registry.server.session.store; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; - import java.util.Collection; import java.util.Iterator; import java.util.Map; @@ -28,8 +24,13 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; + /** - * * @author shangyu.wh * @version $Id: SessionDataStore.java, v 0.1 2017-12-01 18:14 shangyu.wh Exp $ */ @@ -46,10 +47,12 @@ public class SessionDataStore implements DataStore { */ private Map> registry = new ConcurrentHashMap<>(); + /*** index */ private Map> connectIndex = new ConcurrentHashMap<>(); @Override public void add(Publisher publisher) { + Publisher.internPublisher(publisher); write.lock(); try { @@ -66,7 +69,6 @@ public void add(Publisher publisher) { Publisher existingPublisher = publishers.get(publisher.getRegisterId()); if (existingPublisher != null) { - if (existingPublisher.getVersion() != null) { long oldVersion = existingPublisher.getVersion(); Long newVersion = publisher.getVersion(); @@ -100,10 +102,10 @@ public void add(Publisher publisher) { .warn( "There is publisher already exists,version:{},it will be overwrite!Input version:{},info:{}", existingPublisher.getVersion(), publisher.getVersion(), existingPublisher); + removeFromConnectIndex(existingPublisher); } publishers.put(publisher.getRegisterId(), publisher); - - addIndex(publisher); + addToConnectIndex(publisher); } finally { write.unlock(); @@ -118,21 +120,20 @@ public boolean deleteById(String registerId, String dataInfoId) { Map publishers = registry.get(dataInfoId); if (publishers == null) { - LOGGER.error( - "Delete failed because publisher is not registered for dataInfoId: {}", + LOGGER.warn("Delete failed because publisher is not registered for dataInfoId: {}", dataInfoId); return false; } else { Publisher publisherTodelete = publishers.remove(registerId); if (publisherTodelete == null) { - LOGGER.error( + LOGGER.warn( "Delete failed because publisher is not registered for registerId: {}", registerId); return false; } else { - removeIndex(publisherTodelete); + removeFromConnectIndex(publisherTodelete); return true; } } @@ -148,19 +149,21 @@ public Map queryByConnectId(String connectId) { @Override public boolean deleteByConnectId(String connectId) { - write.lock(); try { for (Map map : registry.values()) { for (Iterator it = map.values().iterator(); it.hasNext();) { Publisher publisher = (Publisher) it.next(); if (publisher != null - && connectId.equals(publisher.getSourceAddress().getAddressString())) { + && connectId.equals(WordCache.getInstance().getWordCache( + publisher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString()))) { it.remove(); - invalidateIndex(publisher); } } } + connectIndex.remove(connectId); return true; } catch (Exception e) { LOGGER.error("Delete publisher by connectId {} error!", connectId, e); @@ -189,12 +192,17 @@ public Publisher queryById(String registerId, String dataInfoId) { Map publishers = registry.get(dataInfoId); if (publishers == null) { - LOGGER.error("Publisher is not registered for dataInfoId: {}", dataInfoId); + LOGGER.warn("Publisher is not registered for dataInfoId: {}", dataInfoId); return null; } return publishers.get(registerId); } + @Override + public Collection getStoreDataInfoIds() { + return registry.keySet(); + } + @Override public long count() { AtomicLong count = new AtomicLong(0); @@ -204,14 +212,11 @@ public long count() { return count.get(); } - private void addIndex(Publisher publisher) { - - addConnectIndex(publisher); - } - - private void addConnectIndex(Publisher publisher) { + private void addToConnectIndex(Publisher publisher) { + String connectId = WordCache.getInstance().getWordCache( + publisher.getSourceAddress().getAddressString() + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString()); - String connectId = publisher.getSourceAddress().getAddressString(); Map publisherMap = connectIndex.get(connectId); if (publisherMap == null) { Map newPublisherMap = new ConcurrentHashMap<>(); @@ -224,12 +229,10 @@ private void addConnectIndex(Publisher publisher) { publisherMap.put(publisher.getRegisterId(), publisher); } - private void removeIndex(Publisher publisher) { - removeConnectIndex(publisher); - } - - private void removeConnectIndex(Publisher publisher) { - String connectId = publisher.getSourceAddress().getAddressString(); + private void removeFromConnectIndex(Publisher publisher) { + String connectId = WordCache.getInstance().getWordCache( + publisher.getSourceAddress().getAddressString() + ValueConstants.CONNECT_ID_SPLIT + + publisher.getTargetAddress().getAddressString()); Map publisherMap = connectIndex.get(connectId); if (publisherMap != null) { publisherMap.remove(publisher.getRegisterId()); @@ -238,13 +241,8 @@ private void removeConnectIndex(Publisher publisher) { } } - private void invalidateIndex(Publisher publisher) { - String connectId = publisher.getSourceAddress().getAddressString(); - invalidateConnectIndex(connectId); - } - - private void invalidateConnectIndex(String connectId) { - connectIndex.remove(connectId); + @Override + public Map> getConnectPublishers() { + return connectIndex; } - } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionInterests.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionInterests.java index 68341371e..d5d1415e0 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionInterests.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionInterests.java @@ -16,26 +16,31 @@ */ package com.alipay.sofa.registry.server.session.store; -import com.alipay.sofa.registry.common.model.store.Subscriber; -import com.alipay.sofa.registry.core.model.ScopeEnum; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import com.alipay.sofa.registry.server.session.cache.SubscriberResult; -import com.alipay.sofa.registry.util.VersionsMapUtils; -import org.springframework.beans.factory.annotation.Autowired; - import java.net.InetSocketAddress; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import org.springframework.beans.factory.annotation.Autowired; + +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.core.model.ScopeEnum; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.cache.SubscriberResult; +import com.alipay.sofa.registry.util.VersionsMapUtils; +import com.google.common.collect.Lists; /** - * * @author shangyu.wh * @version $Id: AbstractSessionInterests.java, v 0.1 2017-11-30 20:42 shangyu.wh Exp $ */ @@ -71,6 +76,7 @@ public class SessionInterests implements Interests, ReSubscribers { @Override public void add(Subscriber subscriber) { + Subscriber.internSubscriber(subscriber); write.lock(); try { @@ -89,6 +95,10 @@ public void add(Subscriber subscriber) { if (existingSubscriber != null) { LOGGER.warn("There is subscriber already exists,it will be overwrite! {}", existingSubscriber); + if (sessionServerConfig.isStopPushSwitch()) { + deleteReSubscriber(existingSubscriber); + } + invalidateIndex(existingSubscriber); } subscribers.put(subscriber.getRegisterId(), subscriber); @@ -146,7 +156,10 @@ public boolean deleteByConnectId(String connectId) { for (Map map : interests.values()) { for (Iterator it = map.values().iterator(); it.hasNext();) { Subscriber subscriber = (Subscriber) it.next(); - if (connectId.equals(subscriber.getSourceAddress().getAddressString())) { + if (connectId.equals(WordCache.getInstance().getWordCache( + subscriber.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + subscriber.getTargetAddress().getAddressString()))) { it.remove(); if (sessionServerConfig.isStopPushSwitch()) { @@ -157,6 +170,8 @@ public boolean deleteByConnectId(String connectId) { } } } + //force remove connectId + invalidateConnectIndex(connectId); return true; } catch (Exception e) { LOGGER.error("Delete subscriber by connectId {} error!", connectId, e); @@ -180,6 +195,16 @@ public Map queryByConnectId(String connectId) { return connectIndex.get(connectId); } + public Subscriber queryById(String registerId, String dataInfoId) { + + Map subscribers = interests.get(dataInfoId); + + if (subscribers == null) { + return null; + } + return subscribers.get(registerId); + } + @Override public Collection getInterests(String dataInfoId) { Map subscribers = interests.get(dataInfoId); @@ -196,8 +221,6 @@ public boolean checkInterestVersions(String dataCenter, String dataInfoId, Long Map subscribers = interests.get(dataInfoId); if (subscribers == null || subscribers.isEmpty()) { - LOGGER.info("There are not Subscriber Existed! Who are interest with dataInfoId {} !", - dataInfoId); return false; } @@ -221,6 +244,7 @@ public boolean checkInterestVersions(String dataCenter, String dataInfoId, Long public boolean checkAndUpdateInterestVersions(String dataCenter, String dataInfoId, Long version) { read.lock(); try { + dataInfoId = WordCache.getInstance().getWordCache(dataInfoId); Map subscribers = interests.get(dataInfoId); @@ -240,13 +264,20 @@ public boolean checkAndUpdateInterestVersions(String dataCenter, String dataInfo dataInfoVersions = newDataInfoVersions; } } - + //set zero + if (version.longValue() == 0l) { + return dataInfoVersions.put(dataInfoId, version) != null; + } return VersionsMapUtils.checkAndUpdateVersions(dataInfoVersions, dataInfoId, version); } finally { read.unlock(); } } + public boolean checkAndUpdateInterestVersionZero(String dataCenter, String dataInfoId) { + return checkAndUpdateInterestVersions(dataCenter, dataInfoId, 0l); + } + @Override public Collection getInterestDataInfoIds() { return interests.keySet(); @@ -263,13 +294,16 @@ private void removeIndex(Subscriber subscriber) { } private void invalidateIndex(Subscriber subscriber) { - invalidateConnectIndex(subscriber.getSourceAddress().getAddressString()); + removeConnectIndex(subscriber); invalidateResultIndex(subscriber); } private void addConnectIndex(Subscriber subscriber) { + String connectId = subscriber.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + subscriber.getTargetAddress().getAddressString(); + connectId = WordCache.getInstance().getWordCache(connectId); - String connectId = subscriber.getSourceAddress().getAddressString(); Map subscriberMap = connectIndex.get(connectId); if (subscriberMap == null) { Map newSubscriberMap = new ConcurrentHashMap<>(); @@ -311,7 +345,9 @@ private void addResultIndex(Subscriber subscriber) { } private void removeConnectIndex(Subscriber subscriber) { - String connectId = subscriber.getSourceAddress().getAddressString(); + String connectId = subscriber.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + subscriber.getTargetAddress().getAddressString(); Map subscriberMap = connectIndex.get(connectId); if (subscriberMap != null) { subscriberMap.remove(subscriber.getRegisterId()); @@ -418,6 +454,11 @@ public void clearReSubscribers() { stopPushInterests.clear(); } + @Override + public Map> getConnectSubscribers() { + return connectIndex; + } + public SessionServerConfig getSessionServerConfig() { return sessionServerConfig; } @@ -425,9 +466,19 @@ public SessionServerConfig getSessionServerConfig() { /** * Setter method for property sessionServerConfig. * - * @param sessionServerConfig value to be assigned to property sessionServerConfig + * @param sessionServerConfig value to be assigned to property sessionServerConfig */ public void setSessionServerConfig(SessionServerConfig sessionServerConfig) { this.sessionServerConfig = sessionServerConfig; } + + @Override + public List getDataCenters() { + if (interestVersions != null) { + return interestVersions.keySet().stream().collect(Collectors.toList()); + } else { + return Lists.newArrayList(); + } + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionWatchers.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionWatchers.java index 381dbebe8..fb48e6f9c 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionWatchers.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/SessionWatchers.java @@ -16,11 +16,6 @@ */ package com.alipay.sofa.registry.server.session.store; -import com.alipay.sofa.registry.common.model.store.Watcher; -import com.alipay.sofa.registry.log.Logger; -import com.alipay.sofa.registry.log.LoggerFactory; -import com.alipay.sofa.registry.util.VersionsMapUtils; - import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -30,8 +25,14 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.store.Watcher; +import com.alipay.sofa.registry.common.model.store.WordCache; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.util.VersionsMapUtils; + /** - * * @author shangyu.wh * @version $Id: SessionWatchers.java, v 0.1 2018-04-17 19:00 shangyu.wh Exp $ */ @@ -60,6 +61,8 @@ public class SessionWatchers implements Watchers { @Override public void add(Watcher watcher) { + Watcher.internWatcher(watcher); + write.lock(); try { Map watcherMap = watchers.get(watcher.getDataInfoId()); @@ -77,6 +80,7 @@ public void add(Watcher watcher) { if (existingWatcher != null) { LOGGER.warn("There is watcher already exists,it will be overwrite! {}", existingWatcher); + removeConnectIndex(existingWatcher); } watcherMap.put(watcher.getRegisterId(), watcher); @@ -96,9 +100,13 @@ public boolean deleteByConnectId(String connectId) { for (Iterator it = map.values().iterator(); it.hasNext();) { Watcher watcher = (Watcher) it.next(); if (watcher != null - && connectId.equals(watcher.getSourceAddress().getAddressString())) { + && connectId.equals(watcher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + watcher.getTargetAddress().getAddressString())) { it.remove(); - invalidateConnectIndex(watcher.getSourceAddress().getAddressString()); + invalidateConnectIndex(watcher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + watcher.getTargetAddress().getAddressString()); } } } @@ -198,8 +206,11 @@ public long count() { } private void addConnectIndex(Watcher watcher) { + String connectId = watcher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + watcher.getTargetAddress().getAddressString(); + connectId = WordCache.getInstance().getWordCache(connectId); - String connectId = watcher.getSourceAddress().getAddressString(); Map subscriberMap = connectIndex.get(connectId); if (subscriberMap == null) { Map newSubscriberMap = new ConcurrentHashMap<>(); @@ -213,7 +224,9 @@ private void addConnectIndex(Watcher watcher) { } private void removeConnectIndex(Watcher watcher) { - String connectId = watcher.getSourceAddress().getAddressString(); + String connectId = watcher.getSourceAddress().getAddressString() + + ValueConstants.CONNECT_ID_SPLIT + + watcher.getTargetAddress().getAddressString(); Map subscriberMap = connectIndex.get(connectId); if (subscriberMap != null) { subscriberMap.remove(watcher.getRegisterId()); @@ -226,4 +239,8 @@ private void invalidateConnectIndex(String connectId) { connectIndex.remove(connectId); } + public Map> getConnectWatchers() { + return connectIndex; + } + } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Watchers.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Watchers.java index b4d401042..194472382 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Watchers.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/store/Watchers.java @@ -19,6 +19,7 @@ import com.alipay.sofa.registry.common.model.store.Watcher; import java.util.Collection; +import java.util.Map; /** * @@ -45,4 +46,6 @@ public interface Watchers extends DataManager { * @return */ boolean checkWatcherVersions(String dataInfoId, Long version); + + Map> getConnectWatchers(); } \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultPublisherHandlerStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultPublisherHandlerStrategy.java index 4307f3859..d8b90decc 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultPublisherHandlerStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultPublisherHandlerStrategy.java @@ -64,6 +64,7 @@ public void handlePublisherRegister(Channel channel, PublisherRegister publisher Publisher publisher = PublisherConverter.convert(publisherRegister); publisher.setProcessId(ip + ":" + port); publisher.setSourceAddress(new URL(channel.getRemoteAddress())); + publisher.setTargetAddress(new URL(channel.getLocalAddress())); if (EventTypeConstants.REGISTER.equals(publisherRegister.getEventType())) { sessionRegistry.register(publisher); } else if (EventTypeConstants.UNREGISTER.equals(publisherRegister.getEventType())) { diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSessionRegistryStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSessionRegistryStrategy.java index 630058f56..e7ab95ae3 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSessionRegistryStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSessionRegistryStrategy.java @@ -16,38 +16,34 @@ */ package com.alipay.sofa.registry.server.session.strategy.impl; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.Subscriber; import com.alipay.sofa.registry.common.model.store.Watcher; import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; -import com.alipay.sofa.registry.server.session.cache.CacheService; -import com.alipay.sofa.registry.server.session.cache.DatumKey; -import com.alipay.sofa.registry.server.session.cache.Key; import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.strategy.SessionRegistryStrategy; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; /** + * @author kezhu.wukz * @author xuanbei * @since 2019/2/15 */ public class DefaultSessionRegistryStrategy implements SessionRegistryStrategy { - private static final Logger LOGGER = LoggerFactory - .getLogger(DefaultSessionRegistryStrategy.class); + private static final Logger LOGGER = LoggerFactory + .getLogger(DefaultSessionRegistryStrategy.class); - private static final Logger CONSOLE_LOGGER = LoggerFactory.getLogger("SESSION-CONSOLE", - "[Task]"); - - private static final Logger taskLogger = LoggerFactory.getLogger( - DefaultSessionRegistryStrategy.class, "[Task]"); + private static final Logger taskLogger = LoggerFactory.getLogger( + DefaultSessionRegistryStrategy.class, "[Task]"); /** * store subscribers @@ -64,9 +60,6 @@ public class DefaultSessionRegistryStrategy implements SessionRegistryStrategy { @Autowired private SessionServerConfig sessionServerConfig; - @Autowired - private CacheService sessionCacheService; - @Override public void doFetchChangDataProcess(Map> dataInfoIdVersions) { //diff dataCenter same dataInfoId sent once fetch on cloud mode @@ -75,11 +68,6 @@ public void doFetchChangDataProcess(Map { if (checkInterestVersions(dataCenter, dataInfoID, version)) { - - //update cache - sessionCacheService.invalidate(new Key( - Key.KeyType.OBJ, DatumKey.class.getName(), new DatumKey(dataInfoID, dataCenter))); - changeDataInfoIds.add(dataInfoID); } }); @@ -110,10 +98,7 @@ private void fireDataChangeCloudTask(String dataInfoId) { @Override public void afterPublisherRegister(Publisher publisher) { - TaskEvent syncPublisherTask = new TaskEvent(publisher, - TaskEvent.TaskType.SYNC_PUBLISHER_TASK); - CONSOLE_LOGGER.info("send " + syncPublisherTask.getTaskType() + " taskEvent:{}", publisher); - taskListenerManager.sendTaskEvent(syncPublisherTask); + } @Override diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberHandlerStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberHandlerStrategy.java index 52eb4de74..abd6c6cda 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberHandlerStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberHandlerStrategy.java @@ -66,6 +66,7 @@ public void handleSubscriberRegister(Channel channel, SubscriberRegister subscri subscriber.setProcessId(ip + ":" + port); subscriber.setSourceAddress(new URL(channel.getRemoteAddress(), BoltChannelUtil .getBoltCustomSerializer(channel))); + subscriber.setTargetAddress(new URL(channel.getLocalAddress())); if (EventTypeConstants.REGISTER.equals(subscriberRegister.getEventType())) { sessionRegistry.register(subscriber); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberMultiFetchTaskStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberMultiFetchTaskStrategy.java index a33637d65..d9291d6c1 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberMultiFetchTaskStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberMultiFetchTaskStrategy.java @@ -16,6 +16,17 @@ */ package com.alipay.sofa.registry.server.session.strategy.impl; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import org.springframework.beans.factory.annotation.Autowired; + import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.dataserver.Datum; @@ -27,6 +38,7 @@ import com.alipay.sofa.registry.log.Logger; import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.cache.CacheAccessException; import com.alipay.sofa.registry.server.session.cache.CacheService; import com.alipay.sofa.registry.server.session.cache.DatumKey; import com.alipay.sofa.registry.server.session.cache.Key; @@ -35,19 +47,12 @@ import com.alipay.sofa.registry.server.session.node.NodeManager; import com.alipay.sofa.registry.server.session.node.NodeManagerFactory; import com.alipay.sofa.registry.server.session.scheduler.task.Constant; +import com.alipay.sofa.registry.server.session.store.Interests; import com.alipay.sofa.registry.server.session.strategy.SubscriberMultiFetchTaskStrategy; import com.alipay.sofa.registry.server.session.utils.DatumUtils; import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskListenerManager; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - /** * @author xuanbei * @since 2019/2/15 @@ -60,6 +65,9 @@ public class DefaultSubscriberMultiFetchTaskStrategy implements SubscriberMultiF private static final Logger LOGGER = LoggerFactory .getLogger(DefaultSubscriberMultiFetchTaskStrategy.class); + @Autowired + private Interests sessionInterests; + @Override public void doSubscriberMultiFetchTask(SessionServerConfig sessionServerConfig, TaskListenerManager taskListenerManager, @@ -69,39 +77,29 @@ public void doSubscriberMultiFetchTask(SessionServerConfig sessionServerConfig, Map datumMap = getDatumsCache(fetchDataInfoId, sessionCacheService); - if (datumMap != null) { - - for (ScopeEnum scopeEnum : ScopeEnum.values()) { - Map> map = getPushSubscribers(scopeEnum, - subscribers); - - if (map != null && !map.isEmpty()) { - for (Map.Entry> entry : map - .entrySet()) { - Map subscriberMap = entry.getValue(); - if (subscriberMap != null && !subscriberMap.isEmpty()) { - Subscriber subscriber = subscriberMap.values().iterator().next(); - boolean isOldVersion = !BaseInfo.ClientVersion.StoreData - .equals(subscriber.getClientVersion()); - - if (isOldVersion) { - fireUserDataPushTaskCloud(entry.getKey(), datumMap, - subscriberMap.values(), subscriber, taskListenerManager); - } else { - fireReceivedDataPushTaskCloud(datumMap, - new ArrayList(subscriberMap.keySet()), subscriber, - taskListenerManager); - } + for (ScopeEnum scopeEnum : ScopeEnum.values()) { + Map> map = getPushSubscribers(scopeEnum, + subscribers); + + if (map != null && !map.isEmpty()) { + for (Map.Entry> entry : map.entrySet()) { + Map subscriberMap = entry.getValue(); + if (subscriberMap != null && !subscriberMap.isEmpty()) { + Subscriber subscriber = subscriberMap.values().iterator().next(); + boolean isOldVersion = !BaseInfo.ClientVersion.StoreData.equals(subscriber + .getClientVersion()); + + if (isOldVersion) { + fireUserDataPushTaskCloud(entry.getKey(), datumMap, + subscriberMap.values(), subscriber, taskListenerManager); + } else { + fireReceivedDataPushTaskCloud(datumMap, + new ArrayList(subscriberMap.keySet()), subscriber, + taskListenerManager); } } } } - - } else { - //TODO EMPTY push empty - LOGGER.error( - "SubscriberMultiFetchTask cloud Get publisher data error,which dataInfoId:{}", - fetchDataInfoId); } } @@ -116,7 +114,18 @@ private Map getDatumsCache(String fetchDataInfoId, CacheService s new DatumKey(fetchDataInfoId, dataCenter))). collect(Collectors.toList()); - Map values = sessionCacheService.getValues(keys); + Map values = null; + try { + values = sessionCacheService.getValues(keys); + } catch (CacheAccessException e) { + // The version is set to 0, so that when session checks the datum versions regularly, it will actively re-query the data. + for (String dataCenter : dataCenters) { + boolean result = sessionInterests.checkAndUpdateInterestVersionZero(dataCenter, fetchDataInfoId); + LOGGER.error(String.format( + "error when access cache, so checkAndUpdateInterestVersionZero(return %s): %s", result, + e.getMessage()), e); + } + } if (values != null) { values.forEach((key, value) -> { @@ -140,8 +149,7 @@ private Map> getPushSubscribers(Scope subscribers.forEach((subscriber) -> { if (subscriber.getScope().equals(scopeEnum)) { - InetSocketAddress address = new InetSocketAddress( - subscriber.getSourceAddress().getIpAddress(), + InetSocketAddress address = new InetSocketAddress(subscriber.getSourceAddress().getIpAddress(), subscriber.getSourceAddress().getPort()); Map map = payload.computeIfAbsent(address, k -> new HashMap<>()); map.put(subscriber.getRegisterId(), subscriber); @@ -237,6 +245,8 @@ private void fireReceivedDataPushTaskCloud(Map datu parameter.put(receivedData, subscriber.getSourceAddress()); TaskEvent taskEvent = new TaskEvent(parameter, TaskEvent.TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); + // setup PUSH_CLIENT_SUBSCRIBERS, which is used in AlipayPushTaskMergeProcessor + taskEvent.setAttribute(Constant.PUSH_CLIENT_SUBSCRIBERS, Lists.newArrayList(subscriber)); taskLogger.info("send {} taskURL:{},taskScope:{}", taskEvent.getTaskType(), subscriber.getSourceAddress(), subscriber.getScope()); taskListenerManager.sendTaskEvent(taskEvent); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberRegisterFetchTaskStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberRegisterFetchTaskStrategy.java index 0c7f1ac91..9ef7fdbe3 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberRegisterFetchTaskStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultSubscriberRegisterFetchTaskStrategy.java @@ -108,8 +108,8 @@ private void firePush(ReceivedData receivedData, Subscriber subscriber, parameter.put(receivedData, subscriber.getSourceAddress()); TaskEvent taskEvent = new TaskEvent(parameter, TaskEvent.TaskType.RECEIVED_DATA_MULTI_PUSH_TASK); - taskLogger.info("send {} taskURL:{},taskScope:{}", taskEvent.getTaskType(), - subscriber.getSourceAddress(), receivedData.getScope()); + taskLogger.info("send {} taskURL:{},taskScope:{},taskId:{}", taskEvent.getTaskType(), + subscriber.getSourceAddress(), receivedData.getScope(), taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -141,9 +141,9 @@ private void fireUserDataElementPushTask(Datum datum, Subscriber subscriber, taskEvent.setAttribute(Constant.PUSH_CLIENT_URL, subscriber.getSourceAddress()); int size = datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={}", + taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},taskId={}", taskEvent.getTaskType(), subscriber.getSourceAddress(), datum.getDataInfoId(), - datum.getDataCenter(), size); + datum.getDataCenter(), size, taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } @@ -161,9 +161,9 @@ private void fireUserDataElementMultiPushTask(Datum datum, Subscriber subscriber int size = datum.getPubMap() != null ? datum.getPubMap().size() : 0; - taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={}", + taskLogger.info("send {} taskURL:{},dataInfoId={},dataCenter={},pubSize={},taskId={}", taskEvent.getTaskType(), subscriber.getSourceAddress(), datum.getDataInfoId(), - datum.getDataCenter(), size); + datum.getDataCenter(), size, taskEvent.getTaskId()); taskListenerManager.sendTaskEvent(taskEvent); } } diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultWatcherHandlerStrategy.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultWatcherHandlerStrategy.java index fb9c7ec62..153694a0b 100644 --- a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultWatcherHandlerStrategy.java +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/DefaultWatcherHandlerStrategy.java @@ -61,6 +61,7 @@ public void handleConfiguratorRegister(Channel channel, + channel.getRemoteAddress().getPort()); watcher.setSourceAddress(new URL(channel.getRemoteAddress(), BoltChannelUtil .getBoltCustomSerializer(channel))); + watcher.setTargetAddress(new URL(channel.getLocalAddress())); if (EventTypeConstants.REGISTER.equals(configuratorRegister.getEventType())) { sessionRegistry.register(watcher); diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/AccessLimitWrapperInterceptor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/AccessLimitWrapperInterceptor.java new file mode 100644 index 000000000..4c6c136a4 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/AccessLimitWrapperInterceptor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import com.alipay.sofa.registry.common.model.store.BaseInfo; +import com.alipay.sofa.registry.common.model.store.StoreData; +import com.alipay.sofa.registry.server.session.limit.AccessLimitService; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * + * @author shangyu.wh + * @version 1.0: AccessLimitWrapperInterceptor.java, v 0.1 2019-08-26 20:29 shangyu.wh Exp $ + */ +public class AccessLimitWrapperInterceptor implements WrapperInterceptor { + + @Autowired + private AccessLimitService accessLimitService; + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) + throws Exception { + + BaseInfo baseInfo = (BaseInfo) invocation.getParameterSupplier().get(); + + if (!accessLimitService.tryAcquire()) { + throw new RuntimeException(String.format( + "Register access limit for session server!dataInfoId=%s,connectId=%s", + baseInfo.getDataInfoId(), baseInfo.getSourceAddress())); + } + + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 0; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/BlacklistWrapperInterceptor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/BlacklistWrapperInterceptor.java new file mode 100644 index 000000000..20e076fd2 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/BlacklistWrapperInterceptor.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import com.alipay.sofa.registry.common.model.store.BaseInfo; +import com.alipay.sofa.registry.common.model.store.StoreData; +import com.alipay.sofa.registry.common.model.store.StoreData.DataType; +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.log.LoggerFactory; +import com.alipay.sofa.registry.server.session.filter.ProcessFilter; +import com.alipay.sofa.registry.server.session.registry.SessionRegistry; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * blacklist filter + * + * @author shangyu.wh + * @version 1.0: BlacklistWrapperInterceptor.java, v 0.1 2019-06-18 22:26 shangyu.wh Exp $ + */ +public class BlacklistWrapperInterceptor implements WrapperInterceptor { + + private static final Logger TASK_LOGGER = LoggerFactory.getLogger(SessionRegistry.class, + "[Task]"); + + @Autowired + private TaskListenerManager taskListenerManager; + + /** + * blacklist filter + */ + @Autowired + private ProcessFilter processFilter; + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) + throws Exception { + + BaseInfo storeData = (BaseInfo) invocation.getParameterSupplier().get(); + + if (processFilter.match(storeData)) { + if (DataType.PUBLISHER == storeData.getDataType()) { + // match blacklist stop pub. + return true; + } + + if (DataType.SUBSCRIBER == storeData.getDataType()) { + fireSubscriberPushEmptyTask((Subscriber) storeData); + return true; + } + } + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 200; + } + + private void fireSubscriberPushEmptyTask(Subscriber subscriber) { + //trigger empty data push + TaskEvent taskEvent = new TaskEvent(subscriber, + TaskEvent.TaskType.SUBSCRIBER_PUSH_EMPTY_TASK); + TASK_LOGGER.info("send " + taskEvent.getTaskType() + " taskEvent:{}", taskEvent); + taskListenerManager.sendTaskEvent(taskEvent); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/ClientCheckWrapperInterceptor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/ClientCheckWrapperInterceptor.java new file mode 100644 index 000000000..d3d80ec8c --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/ClientCheckWrapperInterceptor.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import com.alipay.sofa.registry.common.model.store.BaseInfo; +import com.alipay.sofa.registry.common.model.store.StoreData; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * check connect already existed + * + * @author shangyu.wh + * @version 1.0: ClientCheckWrapperInterceptor.java, v 0.1 2019-06-18 13:53 shangyu.wh Exp $ + */ +public class ClientCheckWrapperInterceptor implements WrapperInterceptor { + + @Autowired + private SessionServerConfig sessionServerConfig; + + @Autowired + private Exchange boltExchange; + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) + throws Exception { + + BaseInfo baseInfo = (BaseInfo) invocation.getParameterSupplier().get(); + + Server sessionServer = boltExchange.getServer(sessionServerConfig.getServerPort()); + + Channel channel = sessionServer.getChannel(baseInfo.getSourceAddress()); + + if (channel == null) { + throw new RuntimeException(String.format( + "Register address %s has not connected session server!", + baseInfo.getSourceAddress())); + } + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 100; + } +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/Wrapper.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/Wrapper.java new file mode 100644 index 000000000..19090af52 --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/Wrapper.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import java.util.function.Supplier; + +/** + * + * @author shangyu.wh + * @version 1.0: Wrapper.java, v 0.1 2019-06-18 19:33 shangyu.wh Exp $ + */ +public interface Wrapper { + + R call(); + + Supplier getParameterSupplier(); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptor.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptor.java new file mode 100644 index 000000000..f23478ffe --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptor.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +/** + * + * @author shangyu.wh + * @version 1.0: WrapperInterceptor.java, v 0.1 2019-06-18 11:45 shangyu.wh Exp $ + */ +public interface WrapperInterceptor { + + /** + * invoke inside function + * + * @param invocation + * @return + * @throws Exception + */ + R invokeCodeWrapper(WrapperInvocation invocation) throws Exception; + + /** + * Interceptor order + * @return + */ + int getOrder(); +} \ No newline at end of file diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptorManager.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptorManager.java new file mode 100644 index 000000000..009f6876f --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInterceptorManager.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * + * @author shangyu.wh + * @version 1.0: WrapperInterceptorManager.java, v 0.1 2019-06-18 14:51 shangyu.wh Exp $ + */ +public class WrapperInterceptorManager { + + private List interceptorChain = new CopyOnWriteArrayList<>(); + + public int addInterceptor(WrapperInterceptor item){ + // The index of the search key, if it is contained in the list; otherwise, (-(insertion point) - 1) + final int index = Collections.binarySearch(interceptorChain, item, + (o1, o2) -> { + if (o1 == null && o2 == null) { + return 0; + } else if (o1 == null) { + return 1; + } else if (o2 == null) { + return -1; + } + + if (o1.getOrder() < o2.getOrder()) { + return -1; + } else if (o1.getOrder() > o2.getOrder()) { + return 1; + } + return 0; + }); + final int insertAt; + if (index < 0) { + insertAt = -(index + 1); + } else { + insertAt = index + 1; + } + + interceptorChain.add(insertAt, item); + return insertAt; + } + + /** + * Getter method for property interceptorChain. + * + * @return property value of interceptorChain + */ + public List getInterceptorChain() { + return interceptorChain; + } +} diff --git a/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInvocation.java b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInvocation.java new file mode 100644 index 000000000..7a736944f --- /dev/null +++ b/server/server/session/src/main/java/com/alipay/sofa/registry/server/session/wrapper/WrapperInvocation.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session.wrapper; + +import java.util.Iterator; +import java.util.List; +import java.util.function.Supplier; + +/** + * + * @author shangyu.wh + * @version 1.0: WrapperInvocation.java, v 0.1 2019-06-18 11:43 shangyu.wh Exp $ + */ +public class WrapperInvocation { + + private final Wrapper target; + + private List interceptorChain; + + private Iterator iterator; + + public WrapperInvocation(Wrapper target, + WrapperInterceptorManager wrapperInterceptorManager) { + this.interceptorChain = wrapperInterceptorManager.getInterceptorChain(); + this.iterator = this.interceptorChain.iterator(); + this.target = target; + } + + public R proceed() throws Exception { + if (iterator.hasNext()) { + WrapperInterceptor interceptor = iterator.next(); + return interceptor.invokeCodeWrapper(this); + } + return target.call(); + } + + /** + * Getter method for property ParameterSupplier. + * + * @return property value of ParameterSupplier + */ + public Supplier getParameterSupplier() { + return target.getParameterSupplier(); + } +} \ No newline at end of file diff --git a/server/server/session/src/main/resources/application.properties b/server/server/session/src/main/resources/application.properties index 9df254384..65522368c 100644 --- a/server/server/session/src/main/resources/application.properties +++ b/server/server/session/src/main/resources/application.properties @@ -1,7 +1,7 @@ spring.main.banner-mode=LOG -#nodes.metaNode=: -#nodes.localDataCenter= -#nodes.localRegion= +nodes.metaNode=DefaultDataCenter:localhost +nodes.localDataCenter=DefaultDataCenter +nodes.localRegion=DEFAULT_ZONE #session.server.logging.level=INFO #session.server.logging.home=/home/admin/logs/registry/session @@ -15,16 +15,18 @@ session.server.sessionServerRegion=DEFAULT_ZONE session.server.sessionServerDataCenter=DefaultDataCenter session.server.syncHeartbeat.fixedDelay=30000 session.server.syncExceptionData.fixedDelay=30000 -session.server.printTask.fixedDelay=30000 +session.server.printTask.fixedDelay=1000 session.server.schedulerCheckVersionTimeout=3 session.server.schedulerCheckVersionFirstDelay=3 session.server.schedulerCheckVersionExpBackOffBound=10 -session.server.schedulerHeartbeatTimeout=30 -session.server.schedulerHeartbeatFirstDelay=30 -session.server.schedulerHeartbeatExpBackOffBound=10 +session.server.schedulerHeartbeatTimeout=3 +session.server.schedulerHeartbeatFirstDelay=3 +session.server.schedulerHeartbeatExpBackOffBound=1 session.server.schedulerFetchDataTimeout=3 session.server.schedulerFetchDataFirstDelay=3 session.server.schedulerFetchDataExpBackOffBound=10 #session.server.invalidForeverZones=; #session.server.invalidIgnoreDataidRegex= -#session.server.pushEmptyDataDataIdPrefixes= \ No newline at end of file +#session.server.pushEmptyDataDataIdPrefixes= +session.server.renewDatumWheelTaskDelaySec=180 +session.server.renewDatumWheelTaskRandomFirstDelaySec=60 \ No newline at end of file diff --git a/server/server/session/src/main/resources/logback-spring.xml b/server/server/session/src/main/resources/logback-spring.xml index 7df0a5104..1c816ac08 100644 --- a/server/server/session/src/main/resources/logback-spring.xml +++ b/server/server/session/src/main/resources/logback-spring.xml @@ -71,6 +71,26 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${SESSION_LOG_HOME}/registry-renew.log + + ${SESSION_LOG_HOME}/registry-renew.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -85,7 +105,7 @@ ${SESSION_LOG_HOME}/registry-session.log ${SESSION_LOG_HOME}/registry-session.log.%d{yyyy-MM-dd} - 30 + 3 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -164,8 +184,8 @@ ${SESSION_LOG_HOME}/registry-exchange.log - ${SESSION_LOG_HOME}/registry-exchange.log.%d{yyyy-MM-dd} - 30 + ${SESSION_LOG_HOME}/registry-exchange.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -184,8 +204,8 @@ ${SESSION_LOG_HOME}/registry-push.log - ${SESSION_LOG_HOME}/registry-push.log.%d{yyyy-MM-dd} - 30 + ${SESSION_LOG_HOME}/registry-push.log.%d{yyyy-MM-dd_HH} + 72 [%d{ISO8601}][%p][%t][%c{0}] - %m%n @@ -241,6 +261,26 @@ + + true + + ${LOG_LEVEL} + + + ERROR + DENY + + ${SESSION_LOG_HOME}/registry-metrics.log + + ${SESSION_LOG_HOME}/registry-metrics.log.%d{yyyy-MM-dd} + 30 + + + [%d{ISO8601}][%p][%t][%c{0}] - %m%n + ${LOG_ENCODE} + + + @@ -253,6 +293,11 @@ + + + + + @@ -291,6 +336,17 @@ + + + + + + + + + + + diff --git a/server/server/session/src/main/resources/security/blacklist.txt b/server/server/session/src/main/resources/security/blacklist.txt new file mode 100644 index 000000000..e3413f8d4 --- /dev/null +++ b/server/server/session/src/main/resources/security/blacklist.txt @@ -0,0 +1,78 @@ +org.codehaus.groovy.runtime.MethodClosure +clojure.core$constantly +clojure.main$eval_opt +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactory +com.alibaba.citrus.springext.support.parser.AbstractNamedProxyBeanDefinitionParser$ProxyTargetFactoryImpl +com.alibaba.citrus.springext.util.SpringExtUtil.AbstractProxy +com.alipay.custrelation.service.model.redress.Pair +com.caucho.hessian.test.TestCons +com.mchange.v2.c3p0.JndiRefForwardingDataSource +com.mchange.v2.c3p0.WrapperConnectionPoolDataSource +com.rometools.rome.feed.impl.EqualsBean +com.rometools.rome.feed.impl.ToStringBean +com.sun.jndi.rmi.registry.BindingEnumeration +com.sun.jndi.toolkit.dir.LazySearchEnumerationImpl +com.sun.org.apache.xalan.internal.xsltc.trax.TemplatesImpl +com.sun.rowset.JdbcRowSetImpl +com.sun.xml.internal.bind.v2.runtime.unmarshaller.Base64Data +java.rmi.server.UnicastRemoteObject +java.security.SignedObject +java.util.ServiceLoader$LazyIterator +javax.imageio.ImageIO$ContainsFilter +javax.imageio.spi.ServiceRegistry +javax.management.BadAttributeValueExpException +javax.naming.InitialContext +javax.naming.spi.ObjectFactory +javax.script.ScriptEngineManager +javax.sound.sampled.AudioFormat$Encoding +org.apache.carbondata.core.scan.expression.ExpressionResult +org.apache.commons.dbcp +org.apache.commons.dbcp2 +org.apache.commons.beanutils +org.apache.ibatis.executor.loader.AbstractSerialStateHolder +org.apache.ibatis.executor.loader.CglibSerialStateHolder +org.apache.ibatis.executor.loader.JavassistSerialStateHolder +org.apache.ibatis.executor.loader.cglib.CglibProxyFactory +org.apache.ibatis.executor.loader.javassist.JavassistSerialStateHolder +org.apache.tomcat.dbcp.dbcp.datasources.SharedPoolDataSource +org.apache.wicket.util.upload.DiskFileItem +org.apache.xalan.xsltc.trax.TemplatesImpl +org.apache.xbean.naming.context.ContextUtil$ReadOnlyBinding +org.apache.xpath.XPathContext +org.eclipse.jetty.util.log.LoggerLog +org.geotools.filter.ConstantExpression +org.springframework.aop.aspectj.autoproxy.AspectJAwareAdvisorAutoProxyCreator$PartiallyComparableAdvisorHolder +org.springframework.aop.support.DefaultBeanFactoryPointcutAdvisor +org.springframework.beans.factory.BeanFactory +org.springframework.beans.factory.config.PropertyPathFactoryBean +org.springframework.beans.factory.support.DefaultListableBeanFactory +org.springframework.jndi.support.SimpleJndiBeanFactory +org.springframework.orm.jpa.AbstractEntityManagerFactoryBean +org.springframework.transaction.jta.JtaTransactionManager +org.yaml.snakeyaml.tokens.DirectiveToken +sun.rmi.server.UnicastRef +javax.management.ImmutableDescriptor +org.springframework.jndi.JndiObjectTargetSource +ch.qos.logback.core.db.JNDIConnectionSource +java.beans.Expression +javassist.bytecode +javassist.tools.web.Viewer +javassist.util.proxy.SerializedProxy +org.apache.ibatis.javassist.bytecode +org.apache.ibatis.javassist.tools.web.Viewer +org.apache.ibatis.javassist.util.proxy.SerializedProxy +org.springframework.beans.factory.config.MethodInvokingFactoryBean +com.alibaba.druid +com.sun.corba +com.sun.org.apache.bcel +com.sun.org.apache.xml +com.sun.org.apache.xpath +org.apache.zookeeper.Shell +org.apache.tomcat.dbcp.dbcp.BasicDataSource +net.bytebuddy.dynamic.loading.ByteArrayClassLoader +org.jboss.resteasy.plugins +org.springframework.beans.BeanWrapperImpl$BeanPropertyHandler +org.apache.velocity.runtime.resource.Resource +org.springframework.expression.spel.ast.Indexer$PropertyIndexingValueRef +org.mortbay.log.Slf4jLog +org.springframework.expression.spel.ast.MethodReference$MethodValueRef \ No newline at end of file diff --git a/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/WrapperInvocationTest.java b/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/WrapperInvocationTest.java new file mode 100644 index 000000000..8d97e703f --- /dev/null +++ b/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/WrapperInvocationTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.server.session; + +import com.alipay.sofa.registry.server.session.wrapper.Wrapper; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInterceptor; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInterceptorManager; +import com.alipay.sofa.registry.server.session.wrapper.WrapperInvocation; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static junit.framework.Assert.*; + +/** + * + * @author shangyu.wh + * @version 1.0: WrapperInvocationTest.java, v 0.1 2019-06-18 17:41 shangyu.wh Exp $ + */ +public class WrapperInvocationTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testWrapperInterceptorException() throws Exception { + thrown.expect(IllegalAccessException.class); + thrown.expectMessage("test exception!"); + testAll("EXP"); + } + + @Test + public void testWrapperInterceptorProceed() throws Exception { + assertEquals(testAll("Proceed"), 1); + } + + @Test + public void testWrapperInterceptorAllRun() throws Exception { + assertEquals(testAll("AllRun"), 2); + } + + private int testAll(String input) throws Exception { + + AtomicInteger ret = new AtomicInteger(0); + + WrapperInterceptorManager wrapperInterceptorManager = new WrapperInterceptorManager(); + + wrapperInterceptorManager.addInterceptor(new WrapperInterceptor(){ + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) throws Exception { + System.out.println("0"); + if("EXP".equals(invocation.getParameterSupplier().get())){ + throw new IllegalAccessException("test exception!"); + } + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 0; + } + }); + + wrapperInterceptorManager.addInterceptor(new WrapperInterceptor(){ + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) throws Exception { + System.out.println("1"); + if("Proceed".equals(invocation.getParameterSupplier().get())){ + ret.set(1); + return true; + } + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 1; + } + }); + + wrapperInterceptorManager.addInterceptor(new WrapperInterceptor(){ + + @Override + public Boolean invokeCodeWrapper(WrapperInvocation invocation) throws Exception { + System.out.println("2"); + if("test".equals(invocation.getParameterSupplier().get())){ + return true; + } + return invocation.proceed(); + } + + @Override + public int getOrder() { + return 2; + } + }); + + + + WrapperInvocation wrapperInvocation = new WrapperInvocation(new Wrapper() { + @Override + public Boolean call() { + if("AllRun".equals(getParameterSupplier().get())) { + System.out.println("success"); + ret.set(2); + return true; + } + return null; + } + + @Override + public Supplier getParameterSupplier() { + return ()->input; + } + + }, wrapperInterceptorManager); + + wrapperInvocation.proceed(); + + return ret.get(); + } +} \ No newline at end of file diff --git a/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/store/DataCacheTest.java b/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/store/DataCacheTest.java index 8ca0f69d0..d45a27fd9 100644 --- a/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/store/DataCacheTest.java +++ b/server/server/session/src/test/java/com/alipay/sofa/registry/server/session/store/DataCacheTest.java @@ -16,12 +16,11 @@ */ package com.alipay.sofa.registry.server.session.store; -import com.alipay.sofa.registry.common.model.store.DataInfo; -import com.alipay.sofa.registry.common.model.store.Publisher; -import com.alipay.sofa.registry.common.model.store.Subscriber; -import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.common.model.store.*; import com.alipay.sofa.registry.core.model.ScopeEnum; import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.server.session.bootstrap.CommonConfig; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfigBean; import com.alipay.sofa.registry.server.session.cache.CacheGenerator; import com.alipay.sofa.registry.server.session.cache.CacheService; @@ -54,7 +53,7 @@ public void testGetSub() { sessionInterests.setSessionServerConfig(new SessionServerConfigBean(null)); String dataId = "dataid"; - String connectId = "192.168.1.2:9000"; + String connectId = "192.168.1.2:9000_127.0.0.1:34567"; for (int i = 0; i < 100; i++) { sessionInterests.add(getSub(dataId, ScopeEnum.zone, null, null)); @@ -100,7 +99,7 @@ public void testDeleteSubById() { Map> map = getCacheSub( DataInfo.toDataInfoId(dataId, "instance2", "rpc"), ScopeEnum.zone, sessionInterests); - Assert.assertTrue(getCacheSub(sessionInterests, "192.168.1.9:8000")); + Assert.assertTrue(getCacheSub(sessionInterests, "192.168.1.9:8000_127.0.0.1:34567")); sessionInterests.deleteById("xxregist123", DataInfo.toDataInfoId(dataId, "instance2", "rpc")); @@ -108,7 +107,7 @@ public void testDeleteSubById() { Map> map2 = getCacheSub( DataInfo.toDataInfoId(dataId, "instance2", "rpc"), ScopeEnum.zone, sessionInterests); - Assert.assertFalse(getCacheSub(sessionInterests, "192.168.1.9:8000")); + Assert.assertFalse(getCacheSub(sessionInterests, "192.168.1.9:8000_127.0.0.1:34567")); //map no change Assert.assertEquals(2, map.size()); @@ -134,7 +133,7 @@ public void testGetPub() { ((SessionCacheService) cacheService).setCacheGenerators(cacheGenerators); String dataId = "dataid"; - String connectId = "192.168.1.2:9000"; + String connectId = "192.168.1.2:9000_127.0.0.1:34567"; for (int i = 0; i < 10; i++) { sessionDataStore.add(getPub(dataId, null, null)); @@ -178,7 +177,7 @@ public void testDelPubById() { ((SessionCacheService) cacheService).setCacheGenerators(cacheGenerators); String dataId = "dataid"; - String connectId = "192.168.1.2:9000"; + String connectId = "192.168.1.2:9000_127.0.0.1:34567"; for (int i = 0; i < 10; i++) { sessionDataStore.add(getPub(dataId, null, null)); @@ -187,10 +186,10 @@ public void testDelPubById() { sessionDataStore.add(getPub(dataId, "XXXX", new URL("192.168.1.9", 8000))); Assert.assertTrue(getCachePub(sessionDataStore, connectId)); - Assert.assertTrue(getCachePub(sessionDataStore, "192.168.1.9:8000")); + Assert.assertTrue(getCachePub(sessionDataStore, "192.168.1.9:8000_127.0.0.1:34567")); sessionDataStore.deleteById("XXXX", DataInfo.toDataInfoId(dataId, "instance2", "rpc")); Assert.assertTrue(getCachePub(sessionDataStore, connectId)); - Assert.assertFalse(getCachePub(sessionDataStore, "192.168.1.9:8000")); + Assert.assertFalse(getCachePub(sessionDataStore, "192.168.1.9:8000_127.0.0.1:34567")); } private boolean getCachePub(SessionDataStore sessionDataStore, String connectId) { @@ -230,6 +229,7 @@ private Subscriber getSub(String dataId, ScopeEnum scopeEnum, String registerId, subscriberRegister.setDataInfoId(DataInfo.toDataInfoId(dataId, "instance2", "rpc")); subscriberRegister.setSourceAddress(url == null ? new URL("192.168.1.2", 9000) : url); + subscriberRegister.setTargetAddress(new URL("127.0.0.1", 34567)); return subscriberRegister; } @@ -253,7 +253,221 @@ private Publisher getPub(String dataId, String registerId, URL url) { publisher.setDataInfoId(DataInfo.toDataInfoId(dataId, "instance2", "rpc")); publisher.setSourceAddress(url == null ? new URL("192.168.1.2", 9000) : url); + publisher.setTargetAddress(new URL("127.0.0.1", 34567)); return publisher; } + + @Test + public void testOverwriteSameConnectIdPublisher() { + + SessionDataStore sessionDataStore = new SessionDataStore(); + + Publisher publisher1 = new Publisher(); + publisher1.setDataInfoId("dataInfoId1"); + publisher1.setDataId("dataId1"); + publisher1.setRegisterId("RegisterId1"); + publisher1.setSourceAddress(new URL("192.168.1.1", 12345)); + publisher1.setTargetAddress(new URL("192.168.1.2", 9600)); + + Publisher publisher2 = new Publisher(); + publisher2.setDataInfoId("dataInfoId2"); + publisher2.setDataId("dataId2"); + publisher2.setRegisterId("RegisterId2"); + publisher2.setSourceAddress(new URL("192.168.1.1", 12345)); + publisher2.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionDataStore.add(publisher1); + sessionDataStore.add(publisher2); + + Assert.assertEquals( + sessionDataStore.getConnectPublishers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + sessionDataStore.add(publisher2); + + Assert.assertEquals( + sessionDataStore.getConnectPublishers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + + Publisher publisher3 = new Publisher(); + publisher3.setDataInfoId(publisher1.getDataInfoId()); + publisher3.setDataId(publisher1.getDataId()); + publisher3.setRegisterId(publisher1.getRegisterId()); + publisher3.setSourceAddress(new URL("192.168.1.1", 12346)); + publisher3.setTargetAddress(new URL("192.168.1.2", 9600)); + + Publisher publisher4 = new Publisher(); + publisher4.setDataInfoId(publisher2.getDataInfoId()); + publisher4.setDataId(publisher2.getDataId()); + publisher4.setRegisterId(publisher2.getRegisterId()); + publisher4.setSourceAddress(new URL("192.168.1.1", 12346)); + publisher4.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionDataStore.add(publisher3); + sessionDataStore.add(publisher4); + + Assert.assertEquals( + sessionDataStore.getConnectPublishers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 0); + Assert.assertEquals( + sessionDataStore.getConnectPublishers().get("192.168.1.1:12346_192.168.1.2:9600") + .size(), 2); + + } + + @Test + public void testOverwriteSameConnectIdSubscriber() { + + SessionInterests sessionInterests = new SessionInterests(); + SessionServerConfig config = new SessionServerConfigBean(new CommonConfig()); + config.setStopPushSwitch(false); + sessionInterests.setSessionServerConfig(config); + + Subscriber subscriber1 = new Subscriber(); + subscriber1.setDataInfoId("dataInfoId1"); + subscriber1.setDataId("dataId1"); + subscriber1.setRegisterId("RegisterId1"); + subscriber1.setSourceAddress(new URL("192.168.1.1", 12345)); + subscriber1.setTargetAddress(new URL("192.168.1.2", 9600)); + + Subscriber subscriber2 = new Subscriber(); + subscriber2.setDataInfoId("dataInfoId2"); + subscriber2.setDataId("dataId2"); + subscriber2.setRegisterId("RegisterId2"); + subscriber2.setSourceAddress(new URL("192.168.1.1", 12345)); + subscriber2.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionInterests.add(subscriber1); + sessionInterests.add(subscriber2); + + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + sessionInterests.add(subscriber2); + + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + + Subscriber subscriber3 = new Subscriber(); + subscriber3.setDataInfoId(subscriber1.getDataInfoId()); + subscriber3.setDataId(subscriber1.getDataId()); + subscriber3.setRegisterId(subscriber1.getRegisterId()); + subscriber3.setSourceAddress(new URL("192.168.1.1", 12346)); + subscriber3.setTargetAddress(new URL("192.168.1.2", 9600)); + + Subscriber subscriber4 = new Subscriber(); + subscriber4.setDataInfoId(subscriber2.getDataInfoId()); + subscriber4.setDataId(subscriber2.getDataId()); + subscriber4.setRegisterId(subscriber2.getRegisterId()); + subscriber4.setSourceAddress(new URL("192.168.1.1", 12346)); + subscriber4.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionInterests.add(subscriber3); + sessionInterests.add(subscriber4); + + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12345_192.168.1.2:9600") + .size(), 0); + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12346_192.168.1.2:9600") + .size(), 2); + } + + @Test + public void testOverwriteSameConnectIdWatcher() { + + SessionWatchers sessionWatchers = new SessionWatchers(); + + Watcher watcher1 = new Watcher(); + watcher1.setDataInfoId("dataInfoId1"); + watcher1.setDataId("dataId1"); + watcher1.setRegisterId("RegisterId1"); + watcher1.setSourceAddress(new URL("192.168.1.1", 12345)); + watcher1.setTargetAddress(new URL("192.168.1.2", 9600)); + + Watcher watcher2 = new Watcher(); + watcher2.setDataInfoId("dataInfoId2"); + watcher2.setDataId("dataId2"); + watcher2.setRegisterId("RegisterId2"); + watcher2.setSourceAddress(new URL("192.168.1.1", 12345)); + watcher2.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionWatchers.add(watcher1); + sessionWatchers.add(watcher2); + + Assert.assertEquals(sessionWatchers.queryByConnectId("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + sessionWatchers.add(watcher2); + + Assert.assertEquals(sessionWatchers.queryByConnectId("192.168.1.1:12345_192.168.1.2:9600") + .size(), 2); + + Watcher watcher3 = new Watcher(); + watcher3.setDataInfoId(watcher1.getDataInfoId()); + watcher3.setDataId(watcher1.getDataId()); + watcher3.setRegisterId(watcher1.getRegisterId()); + watcher3.setSourceAddress(new URL("192.168.1.1", 12346)); + watcher3.setTargetAddress(new URL("192.168.1.2", 9600)); + + Watcher watcher4 = new Watcher(); + watcher4.setDataInfoId(watcher2.getDataInfoId()); + watcher4.setDataId(watcher2.getDataId()); + watcher4.setRegisterId(watcher2.getRegisterId()); + watcher4.setSourceAddress(new URL("192.168.1.1", 12346)); + watcher4.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionWatchers.add(watcher3); + sessionWatchers.add(watcher4); + + Assert.assertEquals(sessionWatchers.queryByConnectId("192.168.1.1:12345_192.168.1.2:9600") + .size(), 0); + Assert.assertEquals(sessionWatchers.queryByConnectId("192.168.1.1:12346_192.168.1.2:9600") + .size(), 2); + } + + @Test + public void testSubAndClientOffUnorder() { + SessionInterests sessionInterests = new SessionInterests(); + SessionServerConfig config = new SessionServerConfigBean(null); + sessionInterests.setSessionServerConfig(config); + + Subscriber subscriber1 = new Subscriber(); + subscriber1.setDataInfoId("dataInfoId1"); + subscriber1.setDataId("dataId1"); + subscriber1.setRegisterId("RegisterId1"); + subscriber1.setSourceAddress(new URL("192.168.1.1", 12345)); + subscriber1.setTargetAddress(new URL("192.168.1.2", 9600)); + sessionInterests.add(subscriber1); + + Subscriber subscriber2 = new Subscriber(); + subscriber2.setDataInfoId(subscriber1.getDataInfoId()); + subscriber2.setDataId(subscriber1.getDataId()); + subscriber2.setRegisterId(subscriber1.getRegisterId()); + subscriber2.setSourceAddress(new URL("192.168.1.1", 12346)); + subscriber2.setTargetAddress(new URL("192.168.1.2", 9600)); + + sessionInterests.add(subscriber2); + + sessionInterests.deleteByConnectId(subscriber1.getSourceAddress().getAddressString() + "_" + + subscriber1.getTargetAddress().getAddressString()); + + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12345_192.168.1.2:9600"), + null); + Assert.assertEquals( + sessionInterests.getConnectSubscribers().get("192.168.1.1:12346_192.168.1.2:9600") + .size(), 1); + + Assert.assertEquals( + sessionInterests.querySubscriberIndex(subscriber1.getDataInfoId(), + subscriber1.getScope()).get(new InetSocketAddress("192.168.1.1", 12345)), null); + Assert.assertEquals( + sessionInterests + .querySubscriberIndex(subscriber1.getDataInfoId(), subscriber1.getScope()) + .get(new InetSocketAddress("192.168.1.1", 12346)).size(), 1); + Assert.assertEquals(sessionInterests.getInterests(subscriber1.getDataInfoId()).size(), 1); + Assert.assertTrue(sessionInterests.getInterests(subscriber1.getDataInfoId()).contains( + subscriber2)); + } } \ No newline at end of file diff --git a/server/store/api/pom.xml b/server/store/api/pom.xml index ab28b3dd8..70179d828 100644 --- a/server/store/api/pom.xml +++ b/server/store/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-store - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/server/store/jraft/pom.xml b/server/store/jraft/pom.xml index 188f0d851..7ecb15135 100644 --- a/server/store/jraft/pom.xml +++ b/server/store/jraft/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-store - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftClient.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftClient.java index 84af2be5d..b6485290a 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftClient.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftClient.java @@ -24,8 +24,9 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.rpc.CliClientService; -import com.alipay.sofa.jraft.rpc.impl.AbstractBoltClientService; -import com.alipay.sofa.jraft.rpc.impl.cli.BoltCliClientService; +import com.alipay.sofa.jraft.rpc.impl.AbstractClientService; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient; +import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl; import com.alipay.sofa.registry.jraft.command.ProcessRequest; import com.alipay.sofa.registry.jraft.command.ProcessResponse; import com.alipay.sofa.registry.jraft.handler.NotifyLeaderChangeHandler; @@ -35,39 +36,42 @@ import com.alipay.sofa.registry.remoting.bolt.ConnectionEventAdapter; import com.alipay.sofa.registry.remoting.bolt.SyncUserProcessorAdapter; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** - * * @author shangyu.wh * @version $Id: RaftClient.java, v 0.1 2018-05-16 11:40 shangyu.wh Exp $ */ public class RaftClient { - private static final Logger LOGGER = LoggerFactory.getLogger(RaftClient.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RaftClient.class); + + private CliClientService cliClientService; + private RpcClient rpcClient; + private CliOptions cliOptions; + private String groupId; + private Configuration conf; - private BoltCliClientService cliClientService; - private RpcClient rpcClient; - private CliOptions cliOptions; - private String groupId; - private Configuration conf; + private AtomicBoolean started = new AtomicBoolean(false); - private AtomicBoolean started = new AtomicBoolean(false); + private ThreadPoolExecutor executor; /** * @param groupId - * @param confStr Example: 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083 + * @param confStr Example: 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083 */ - public RaftClient(String groupId, String confStr) { + public RaftClient(String groupId, String confStr, ThreadPoolExecutor executor) { this.groupId = groupId; + this.executor = executor; conf = new Configuration(); if (!conf.parse(confStr)) { throw new IllegalArgumentException("Fail to parse conf:" + confStr); } cliOptions = new CliOptions(); - cliClientService = new BoltCliClientService(); + cliClientService = new CliClientServiceImpl(); } /** @@ -75,15 +79,17 @@ public RaftClient(String groupId, String confStr) { * @param confStr * @param cliClientService */ - public RaftClient(String groupId, String confStr, AbstractBoltClientService cliClientService) { + public RaftClient(String groupId, String confStr, AbstractClientService cliClientService, + ThreadPoolExecutor executor) { this.groupId = groupId; + this.executor = executor; conf = new Configuration(); if (!conf.parse(confStr)) { throw new IllegalArgumentException("Fail to parse conf:" + confStr); } cliOptions = new CliOptions(); - this.cliClientService = (BoltCliClientService) cliClientService; + this.cliClientService = (CliClientService) cliClientService; } /** @@ -96,10 +102,12 @@ public void start() { cliClientService.init(cliOptions); - rpcClient = cliClientService.getRpcClient(); + BoltRpcClient jraftRpcClient = ((BoltRpcClient) ((AbstractClientService) cliClientService) + .getRpcClient()); + rpcClient = jraftRpcClient.getRpcClient(); RaftClientConnectionHandler raftClientConnectionHandler = new RaftClientConnectionHandler( - this); + this, executor); rpcClient.addConnectionEventProcessor(ConnectionEventType.CONNECT, new ConnectionEventAdapter(ConnectionEventType.CONNECT, @@ -113,7 +121,7 @@ public void start() { //reset leader notify NotifyLeaderChangeHandler notifyLeaderChangeHandler = new NotifyLeaderChangeHandler( - groupId, cliClientService); + groupId, cliClientService, executor); rpcClient .registerUserProcessor(new SyncUserProcessorAdapter(notifyLeaderChangeHandler)); @@ -131,6 +139,7 @@ public void shutdown() { /** * repick leader + * * @return */ public PeerId refreshLeader() { @@ -166,6 +175,7 @@ public static PeerId refreshLeader(CliClientService cliClientService, String gro /** * get leader + * * @return */ public PeerId getLeader() { @@ -178,6 +188,7 @@ public PeerId getLeader() { /** * raft client send request + * * @param request * @return */ diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServer.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServer.java index 2aad27198..db60729f4 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServer.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServer.java @@ -17,7 +17,6 @@ package com.alipay.sofa.registry.jraft.bootstrap; import com.alipay.remoting.rpc.RpcClient; -import com.alipay.remoting.rpc.RpcServer; import com.alipay.sofa.jraft.Node; import com.alipay.sofa.jraft.RaftGroupService; import com.alipay.sofa.jraft.conf.Configuration; @@ -25,7 +24,12 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; -import com.alipay.sofa.jraft.rpc.impl.AbstractBoltClientService; +import com.alipay.sofa.jraft.rpc.RpcServer; +import com.alipay.sofa.jraft.rpc.impl.AbstractClientService; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcServer; +import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; +import com.alipay.sofa.jraft.util.StorageOptionsFactory; import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.jraft.command.NotifyLeaderChange; import com.alipay.sofa.registry.jraft.handler.NotifyLeaderChangeHandler; @@ -42,20 +46,29 @@ import com.alipay.sofa.registry.remoting.bolt.BoltServer; import com.alipay.sofa.registry.remoting.bolt.SyncUserProcessorAdapter; import com.alipay.sofa.registry.util.FileUtils; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.IndexType; +import org.rocksdb.RocksDB; +import org.rocksdb.util.SizeUnit; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.*; /** - * * @author shangyu.wh * @version $Id: RaftServer.java, v 0.1 2018-05-16 11:39 shangyu.wh Exp $ */ public class RaftServer { + static { + RocksDB.loadLibrary(); + } + private static final Logger LOGGER = LoggerFactory.getLogger(RaftServer.class); private RaftGroupService raftGroupService; @@ -72,9 +85,11 @@ public class RaftServer { private FollowerProcessListener followerProcessListener; private BoltServer boltServer; + private ThreadPoolExecutor raftExecutor; + private ThreadPoolExecutor raftServerExecutor; + private ThreadPoolExecutor fsmExecutor; /** - * * @param dataPath Example: /tmp/server1 * @param groupId * @param serverIdStr Example: 127.0.0.1:8081 @@ -97,26 +112,28 @@ public RaftServer(String dataPath, String groupId, String serverIdStr, String in /** * start raft server + * * @param raftServerConfig * @throws IOException */ - public void start(RaftServerConfig raftServerConfig) throws IOException { + public void start(RaftServerConfig raftServerConfig, ThreadPoolExecutor executor) + throws IOException { FileUtils.forceMkdir(new File(dataPath)); - serverHandlers.add(new RaftServerHandler(this)); - serverHandlers.add(new RaftServerConnectionHandler()); + serverHandlers.add(new RaftServerHandler(this, raftServerExecutor)); + serverHandlers.add(new RaftServerConnectionHandler(executor)); boltServer = new BoltServer(new URL(NetUtil.getLocalAddress().getHostAddress(), serverId.getPort()), serverHandlers); boltServer.initServer(); - RpcServer rpcServer = boltServer.getRpcServer(); - - RaftRpcServerFactory.addRaftRequestProcessors(rpcServer); + RpcServer rpcServer = new BoltRpcServer(boltServer.getRpcServer()); + RaftRpcServerFactory.addRaftRequestProcessors(rpcServer, raftExecutor, raftExecutor); this.fsm = ServiceStateMachine.getInstance(); + this.fsm.setExecutor(this.fsmExecutor); this.fsm.setLeaderProcessListener(leaderProcessListener); this.fsm.setFollowerProcessListener(followerProcessListener); @@ -131,11 +148,11 @@ public void start(RaftServerConfig raftServerConfig) throws IOException { node.getNodeMetrics().getMetricRegistry(), raftServerConfig.getMetricsLogger()); } - RpcClient raftClient = ((AbstractBoltClientService) (((NodeImpl) node).getRpcService())) - .getRpcClient(); + RpcClient raftClient = ((BoltRpcClient) ((AbstractClientService) (((NodeImpl) node) + .getRpcService())).getRpcClient()).getRpcClient(); NotifyLeaderChangeHandler notifyLeaderChangeHandler = new NotifyLeaderChangeHandler( - groupId, null); + groupId, null, executor); raftClient.registerUserProcessor(new SyncUserProcessorAdapter(notifyLeaderChangeHandler)); } @@ -146,6 +163,9 @@ public void shutdown() { if (raftGroupService != null) { this.raftGroupService.shutdown(); } + raftExecutor.shutdown(); + raftServerExecutor.shutdown(); + fsmExecutor.shutdown(); } private NodeOptions initNodeOptions(RaftServerConfig raftServerConfig) { @@ -170,11 +190,30 @@ private NodeOptions initNodeOptions(RaftServerConfig raftServerConfig) { nodeOptions.setEnableMetrics(raftServerConfig.isEnableMetrics()); } + // See https://github.com/sofastack/sofa-jraft/pull/156 + final BlockBasedTableConfig conf = new BlockBasedTableConfig() // + // Begin to use partitioned index filters + // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters#how-to-use-it + .setIndexType(IndexType.kTwoLevelIndexSearch) // + .setFilter(new BloomFilter(16, false)) // + .setPartitionFilters(true) // + .setMetadataBlockSize(8 * SizeUnit.KB) // + .setCacheIndexAndFilterBlocks(false) // + .setCacheIndexAndFilterBlocksWithHighPriority(true) // + .setPinL0FilterAndIndexBlocksInCache(true) // + // End of partitioned index filters settings. + .setBlockSize(4 * SizeUnit.KB)// + .setBlockCacheSize(raftServerConfig.getRockDBCacheSize() * SizeUnit.MB) // + .setCacheNumShardBits(8); + + StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class, conf); + return nodeOptions; } /** * Redirect request to new leader + * * @return */ public String redirect() { @@ -189,6 +228,7 @@ public String redirect() { /** * send notify + * * @param leader * @param sender */ @@ -240,7 +280,7 @@ public Node getNode() { /** * Setter method for property leaderProcessListener. * - * @param leaderProcessListener value to be assigned to property leaderProcessListener + * @param leaderProcessListener value to be assigned to property leaderProcessListener */ public void setLeaderProcessListener(LeaderProcessListener leaderProcessListener) { this.leaderProcessListener = leaderProcessListener; @@ -249,10 +289,30 @@ public void setLeaderProcessListener(LeaderProcessListener leaderProcessListener /** * Setter method for property followerProcessListener. * - * @param followerProcessListener value to be assigned to property followerProcessListener + * @param followerProcessListener value to be assigned to property followerProcessListener */ public void setFollowerProcessListener(FollowerProcessListener followerProcessListener) { this.followerProcessListener = followerProcessListener; } + /** + * Getter method for property serverHandlers. + * + * @return property value of serverHandlers + */ + public List getServerHandlers() { + return serverHandlers; + } + + public void setRaftExecutor(ThreadPoolExecutor executor) { + this.raftExecutor = executor; + } + + public void setRaftServerExecutor(ThreadPoolExecutor executor) { + this.raftServerExecutor = executor; + } + + public void setFsmExecutor(ThreadPoolExecutor executor) { + this.fsmExecutor = executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServerConfig.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServerConfig.java index 9e68e1e66..f739bc81b 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServerConfig.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/RaftServerConfig.java @@ -50,6 +50,8 @@ public class RaftServerConfig { */ private int snapshotIntervalSecs = 3600; + private int RockDBCacheSize = 64; //64M + private Logger metricsLogger; /** @@ -141,4 +143,22 @@ public Logger getMetricsLogger() { public void setMetricsLogger(Logger metricsLogger) { this.metricsLogger = metricsLogger; } + + /** + * Getter method for property RockDBCacheSize. + * + * @return property value of RockDBCacheSize + */ + public int getRockDBCacheSize() { + return RockDBCacheSize; + } + + /** + * Setter method for property RockDBCacheSize. + * + * @param RockDBCacheSize value to be assigned to property RockDBCacheSize + */ + public void setRockDBCacheSize(int rockDBCacheSize) { + RockDBCacheSize = rockDBCacheSize; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/ServiceStateMachine.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/ServiceStateMachine.java index 7f869b7b2..62473d7ac 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/ServiceStateMachine.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/bootstrap/ServiceStateMachine.java @@ -24,7 +24,6 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; -import com.alipay.sofa.jraft.util.Utils; import com.alipay.sofa.registry.jraft.command.ProcessRequest; import com.alipay.sofa.registry.jraft.command.ProcessResponse; import com.alipay.sofa.registry.jraft.processor.FollowerProcessListener; @@ -46,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; /** @@ -64,6 +64,8 @@ public class ServiceStateMachine extends StateMachineAdapter { private static volatile ServiceStateMachine instance; + private ThreadPoolExecutor executor; + /** * get instance of ServiceStateMachine * @return @@ -153,7 +155,7 @@ public void onSnapshotSave(final SnapshotWriter writer, final Closure done) { } }); } - Utils.runInThread(() -> { + this.executor.execute(() -> { String errors = null; outer: for (Map.Entry entry : snapshotProcessors.entrySet()) { @@ -231,7 +233,7 @@ public boolean onSnapshotLoad(SnapshotReader reader) { public void onLeaderStart(long term) { this.leaderTerm.set(term); if (leaderProcessListener != null) { - Utils.runInThread(() -> leaderProcessListener.startProcess()); + this.executor.execute(() -> leaderProcessListener.startProcess()); } super.onLeaderStart(term); } @@ -240,7 +242,7 @@ public void onLeaderStart(long term) { public void onLeaderStop(Status status) { this.leaderTerm.set(-1); if (leaderProcessListener != null) { - Utils.runInThread(() -> leaderProcessListener.stopProcess()); + this.executor.execute(() -> leaderProcessListener.stopProcess()); } super.onLeaderStop(status); } @@ -250,7 +252,7 @@ public void onStopFollowing(LeaderChangeContext ctx) { this.followerTerm.set(-1); if (followerProcessListener != null) { - Utils.runInThread(() -> followerProcessListener.stopProcess(ctx.getLeaderId())); + this.executor.execute(() -> followerProcessListener.stopProcess(ctx.getLeaderId())); } super.onStopFollowing(ctx); } @@ -260,7 +262,7 @@ public void onStartFollowing(LeaderChangeContext ctx) { this.followerTerm.set(1); if (followerProcessListener != null) { - Utils.runInThread(() -> followerProcessListener.startProcess(ctx.getLeaderId())); + this.executor.execute(() -> followerProcessListener.startProcess(ctx.getLeaderId())); } super.onStartFollowing(ctx); } @@ -282,4 +284,8 @@ public void setLeaderProcessListener(LeaderProcessListener leaderProcessListener public void setFollowerProcessListener(FollowerProcessListener followerProcessListener) { this.followerProcessListener = followerProcessListener; } + + public void setExecutor(ThreadPoolExecutor executor) { + this.executor = executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/NotifyLeaderChangeHandler.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/NotifyLeaderChangeHandler.java index e2ebae18d..7c6a72640 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/NotifyLeaderChangeHandler.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/NotifyLeaderChangeHandler.java @@ -25,6 +25,9 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + /** * * @author shangyu.wh @@ -38,13 +41,17 @@ public class NotifyLeaderChangeHandler implements ChannelHandler { private CliClientService clientService; + private ThreadPoolExecutor executor; + /** * constructor * @param groupId */ - public NotifyLeaderChangeHandler(String groupId, CliClientService clientService) { + public NotifyLeaderChangeHandler(String groupId, CliClientService clientService, + ThreadPoolExecutor executor) { this.groupId = groupId; this.clientService = clientService; + this.executor = executor; } @Override @@ -94,4 +101,9 @@ public HandlerType getType() { public Class interest() { return NotifyLeaderChange.class; } + + @Override + public Executor getExecutor() { + return executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftClientConnectionHandler.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftClientConnectionHandler.java index 2a605e2fd..cf345712d 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftClientConnectionHandler.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftClientConnectionHandler.java @@ -23,6 +23,10 @@ import com.alipay.sofa.registry.log.LoggerFactory; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; +import sun.nio.ch.ThreadPool; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; /** * @@ -35,12 +39,15 @@ public class RaftClientConnectionHandler implements ChannelHandler { private RaftClient raftClient; + private ThreadPoolExecutor executor; + /** * constructor * @param raftClient */ - public RaftClientConnectionHandler(RaftClient raftClient) { + public RaftClientConnectionHandler(RaftClient raftClient, ThreadPoolExecutor executor) { this.raftClient = raftClient; + this.executor = executor; } @Override @@ -87,4 +94,9 @@ public HandlerType getType() { public Class interest() { return null; } + + @Override + public Executor getExecutor() { + return executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerConnectionHandler.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerConnectionHandler.java index 96c60f51d..4199bce54 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerConnectionHandler.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerConnectionHandler.java @@ -21,6 +21,9 @@ import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.remoting.ChannelHandler; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + /** * * @author shangyu.wh @@ -30,6 +33,12 @@ public class RaftServerConnectionHandler implements ChannelHandler { private static final Logger LOGGER = LoggerFactory.getLogger(RaftServerConnectionHandler.class); + private ThreadPoolExecutor executor; + + public RaftServerConnectionHandler(ThreadPoolExecutor executor) { + this.executor = executor; + } + @Override public void connected(Channel channel) { if (channel != null && channel.isConnected()) { @@ -70,4 +79,9 @@ public HandlerType getType() { public Class interest() { return null; } + + @Override + public Executor getExecutor() { + return executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerHandler.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerHandler.java index 12bbba9a6..9dc57bb8a 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerHandler.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/handler/RaftServerHandler.java @@ -36,6 +36,8 @@ import java.io.IOException; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; /** * @@ -48,12 +50,15 @@ public class RaftServerHandler implements ChannelHandler { protected RaftServer raftServer; + private ThreadPoolExecutor executor; + /** * constructor * @param raftServer */ - public RaftServerHandler(RaftServer raftServer) { + public RaftServerHandler(RaftServer raftServer, ThreadPoolExecutor executor) { this.raftServer = raftServer; + this.executor = executor; } @Override @@ -159,4 +164,9 @@ public Object reply(Channel channel, Object message) { public InvokeType getInvokeType() { return InvokeType.ASYNC; } + + @Override + public Executor getExecutor() { + return executor; + } } \ No newline at end of file diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/Processor.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/Processor.java index a395a365d..fab3c1cd1 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/Processor.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/Processor.java @@ -28,6 +28,7 @@ import java.lang.reflect.Method; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * @@ -36,12 +37,16 @@ */ public class Processor { - private static final Logger LOG = LoggerFactory - .getLogger(Processor.class); + private static final Logger LOG = LoggerFactory + .getLogger(Processor.class); - private Map> workerMethods = new HashMap<>(); + private Map> workerMethods = new HashMap<>(); - private Map workers = new HashMap<>(); + private Map workers = new HashMap<>(); + + private Map methodHandleMap = new ConcurrentHashMap<>(); + + private final static String SERVICE_METHOD_SPLIT = "#@#"; private static volatile Processor instance; @@ -97,15 +102,24 @@ public ProcessResponse process(ProcessRequest request) { for (int i = 0; i < sig.length; i++) { methodKeyBuffer.append(sig[i]); } - Method appServiceMethod = workerMethods.get(serviceId).get(methodKeyBuffer.toString()); + String methodKey = methodKeyBuffer.toString(); + Method appServiceMethod = workerMethods.get(serviceId).get(methodKey); if (appServiceMethod == null) { LOG.error("Can not find method {} from processor by serviceId {}", methodName, serviceId); throw new NoSuchMethodException("Can not find method from processor！"); } - Object[] methodArg = request.getMethodArgs(); - MethodHandle methodHandle = MethodHandles.lookup().unreflect(appServiceMethod); + String methodHandleKey = getMethodHandleKey(serviceId, methodKey); + + MethodHandle methodHandle = methodHandleMap.computeIfAbsent(methodHandleKey,k-> { + try { + return MethodHandles.lookup().unreflect(appServiceMethod); + } catch (IllegalAccessException e) { + throw new RuntimeException("Process service request lookup method error!",e); + } + }); + Object ret = methodHandle.bindTo(target).invokeWithArguments(methodArg); if (ret != null) { return ProcessResponse.ok(ret).build(); @@ -133,7 +147,23 @@ public ProcessResponse process(Method method, ProcessRequest request) { try { Object[] methodArg = request.getMethodArgs(); - MethodHandle methodHandle = MethodHandles.lookup().unreflect(method); + + StringBuilder methodKeyBuffer = new StringBuilder(); + methodKeyBuffer.append(methodName); + String[] sig = request.getMethodArgSigs(); + for (int i = 0; i < sig.length; i++) { + methodKeyBuffer.append(sig[i]); + } + String methodKey = methodKeyBuffer.toString(); + String methodHandleKey = getMethodHandleKey(serviceId, methodKey); + MethodHandle methodHandle = methodHandleMap.computeIfAbsent(methodHandleKey,k-> { + try { + return MethodHandles.lookup().unreflect(method); + } catch (IllegalAccessException e) { + throw new RuntimeException("Process service request lookup method error!",e); + } + }); + Object ret = methodHandle.bindTo(target).invokeWithArguments(methodArg); if (ret != null) { return ProcessResponse.ok(ret).build(); @@ -174,6 +204,42 @@ public Method getWorkMethod(ProcessRequest request) { } } + public MethodHandle getWorkMethodHandle(ProcessRequest request) { + String methodName = request.getMethodName(); + String serviceId = request.getServiceName(); + try { + + StringBuilder methodKeyBuffer = new StringBuilder(); + methodKeyBuffer.append(methodName); + String[] sig = request.getMethodArgSigs(); + for (int i = 0; i < sig.length; i++) { + methodKeyBuffer.append(sig[i]); + } + String methodKey = methodKeyBuffer.toString(); + Method appServiceMethod = workerMethods.get(serviceId).get(methodKey); + if (appServiceMethod == null) { + LOG.error("Can not find method {} from processor by serviceId {}", methodName, + serviceId); + throw new NoSuchMethodException("Can not find method from processor！"); + } + + String methodHandleKey = getMethodHandleKey(serviceId, methodKey); + + MethodHandle methodHandle = methodHandleMap.computeIfAbsent(methodHandleKey,k-> { + try { + return MethodHandles.lookup().unreflect(appServiceMethod); + } catch (IllegalAccessException e) { + throw new RuntimeException("Process service request lookup method error!",e); + } + }); + return methodHandle; + } catch (Exception e) { + LOG.error("Process request {} get WorkMethod error!", request, e); + throw new RuntimeException(String.format("Process request %s get WorkMethod error!", + request)); + } + } + public Map getWorkers() { return workers; } @@ -185,4 +251,8 @@ public boolean isLeaderReadMethod(Method method) { return false; } + public String getMethodHandleKey(String serviceId, String methodKey) { + return serviceId + SERVICE_METHOD_SPLIT + methodKey; + } + } diff --git a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/ProxyHandler.java b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/ProxyHandler.java index f96098cf2..09b9e63c5 100644 --- a/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/ProxyHandler.java +++ b/server/store/jraft/src/main/java/com/alipay/sofa/registry/jraft/processor/ProxyHandler.java @@ -22,7 +22,6 @@ import com.alipay.sofa.registry.log.LoggerFactory; import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; @@ -87,9 +86,7 @@ private Object doInvokeMethod(ProcessRequest request) { serviceId)); } - Method method = Processor.getInstance().getWorkMethod(request); - - MethodHandle methodHandle = MethodHandles.lookup().unreflect(method); + MethodHandle methodHandle = Processor.getInstance().getWorkMethodHandle(request); return methodHandle.bindTo(target).invokeWithArguments(request.getMethodArgs()); } catch (Throwable e) { LOGGER.error("Directly invoke read only service {} method {} error!", diff --git a/server/store/jraft/src/test/java/TestServiceStateMachine.java b/server/store/jraft/src/test/java/TestServiceStateMachine.java new file mode 100644 index 000000000..beffd914d --- /dev/null +++ b/server/store/jraft/src/test/java/TestServiceStateMachine.java @@ -0,0 +1,565 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Iterator; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.alipay.sofa.registry.jraft.bootstrap.ServiceStateMachine; +import com.alipay.sofa.registry.jraft.command.ProcessRequest; +import com.alipay.sofa.registry.jraft.command.ProcessResponse; +import com.alipay.sofa.registry.jraft.processor.FollowerProcessListener; +import com.alipay.sofa.registry.jraft.processor.LeaderProcessListener; +import com.alipay.sofa.registry.jraft.processor.LeaderTaskClosure; +import com.alipay.sofa.registry.jraft.processor.Processor; +import com.alipay.sofa.registry.jraft.processor.SnapshotProcess; +import com.caucho.hessian.io.Hessian2Output; +import com.caucho.hessian.io.SerializerFactory; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.OneofDescriptor; +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import com.google.protobuf.UnknownFieldSet; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * + * @author shangyu.wh + * @version 1.0: TestServiceStateMachine.java, v 0.1 2019-08-01 15:16 shangyu.wh Exp $ + */ +public class TestServiceStateMachine { + + @Test + public void testApply() { + ServiceStateMachine serviceStateMachine = ServiceStateMachine.getInstance(); + + Processor processor = Processor.getInstance(); + + processor.addWorker(TestServiceStateMachine.class.getSimpleName(), + TestServiceStateMachine.class, new TestServiceStateMachine()); + + AtomicInteger count = new AtomicInteger(); + + LeaderTaskClosure leaderTaskClosure = new LeaderTaskClosure(); + ProcessRequest processRequest = new ProcessRequest(); + processRequest + .setMethodArgs(new Object[] { TestServiceStateMachine.class.getSimpleName() }); + processRequest.setMethodArgSigs(new String[] { "java.lang.String" }); + processRequest.setMethodName("testMethod"); + processRequest.setServiceName(TestServiceStateMachine.class.getSimpleName()); + leaderTaskClosure.setRequest(processRequest); + + serviceStateMachine.onApply(new Iterator() { + + @Override + public boolean hasNext() { + if (count.get() > 0) + return false; + return true; + } + + @Override + public ByteBuffer next() { + count.getAndIncrement(); + return null; + } + + @Override + public ByteBuffer getData() { + return ByteBuffer.allocate(10); + } + + @Override + public long getIndex() { + return 0; + } + + @Override + public long getTerm() { + return 0; + } + + @Override + public Closure done() { + return leaderTaskClosure; + } + + @Override + public void setErrorAndRollback(long ntail, Status st) { + + } + }); + + Assert.assertEquals(TestServiceStateMachine.class.getSimpleName(), + ((ProcessResponse) leaderTaskClosure.getResponse()).getEntity()); + } + + @Test + public void testApply2() { + ServiceStateMachine serviceStateMachine = ServiceStateMachine.getInstance(); + + Processor processor = Processor.getInstance(); + + processor.addWorker(TestServiceStateMachine.class.getSimpleName(), + TestServiceStateMachine.class, new TestServiceStateMachine()); + + AtomicInteger count = new AtomicInteger(); + + LeaderTaskClosure leaderTaskClosure = new LeaderTaskClosure(); + ProcessRequest processRequest = new ProcessRequest(); + processRequest + .setMethodArgs(new Object[] { TestServiceStateMachine.class.getSimpleName() }); + processRequest.setMethodArgSigs(new String[] { "java.lang.String" }); + processRequest.setMethodName("testMethod"); + processRequest.setServiceName(TestServiceStateMachine.class.getSimpleName()); + leaderTaskClosure.setRequest(processRequest); + + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + Hessian2Output hessianOutput = new Hessian2Output(byteStream); + SerializerFactory serializerFactory = new SerializerFactory(); + hessianOutput.setSerializerFactory(serializerFactory); + try { + hessianOutput.writeObject(processRequest); + hessianOutput.close(); + } catch (IOException e) { + } + + serviceStateMachine.onApply(new Iterator() { + + @Override + public boolean hasNext() { + if (count.get() > 0) + return false; + return true; + } + + @Override + public ByteBuffer next() { + count.getAndIncrement(); + return null; + } + + @Override + public ByteBuffer getData() { + byte[] cmdBytes = byteStream.toByteArray(); + + ByteBuffer data = ByteBuffer.allocate(cmdBytes.length); + data.put(cmdBytes); + data.flip(); + return data; + } + + @Override + public long getIndex() { + return 0; + } + + @Override + public long getTerm() { + return 0; + } + + @Override + public Closure done() { + return null; + } + + @Override + public void setErrorAndRollback(long ntail, Status st) { + + } + }); + + } + + @Test + public void testOnSnapshotSave() throws InterruptedException { + ServiceStateMachine serviceStateMachine = ServiceStateMachine.getInstance(); + Processor processor = Processor.getInstance(); + LeaderTaskClosure leaderTaskClosure = new LeaderTaskClosure(); + AtomicReference status = new AtomicReference<>(); + serviceStateMachine.setExecutor(new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>(20))); + leaderTaskClosure.setDone(statusIn -> { + status.set(statusIn); + }); + + SnapshotProcess process = new SnapshotProcess(){ + + @Override + public boolean save(String path) { + return true; + } + + @Override + public boolean load(String path) { + return true; + } + + @Override + public SnapshotProcess copy() { + return this; + } + + @Override + public Set getSnapshotFileNames() { + return new HashSet<>(Arrays.asList(new String[]{"aaa","sss"})); + } + }; + processor.addWorker(process.getClass().getSimpleName(),process.getClass(),process); + + serviceStateMachine.onSnapshotSave(new SnapshotWriter(){ + + @Override + public void close() throws IOException { + + } + + @Override + public String getPath() { + return "path"; + } + + @Override + public Set listFiles() { + return null; + } + + @Override + public Message getFileMeta(String fileName) { + return null; + } + + @Override + public boolean init(Void opts) { + return false; + } + + @Override + public void shutdown() { + + } + + @Override + public boolean saveMeta(SnapshotMeta meta) { + return false; + } + + public boolean addFile(String fileName){ + return true; + } + + @Override + public boolean addFile(String fileName, Message fileMeta) { + return true; + } + + @Override + public boolean removeFile(String fileName) { + return false; + } + + @Override + public void close(boolean keepDataOnError) throws IOException { + + } + },leaderTaskClosure); + + TimeUnit.MILLISECONDS.sleep(1000); + + Assert.assertTrue(status.get().isOk()); + } + + @Test + public void testOnSnapshotLoad() throws InterruptedException { + ServiceStateMachine serviceStateMachine = ServiceStateMachine.getInstance(); + serviceStateMachine.setExecutor(new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(20))); + Processor processor = Processor.getInstance(); + SnapshotProcess process = new SnapshotProcess() { + + @Override + public boolean save(String path) { + return true; + } + + @Override + public boolean load(String path) { + return true; + } + + @Override + public SnapshotProcess copy() { + return this; + } + + @Override + public Set getSnapshotFileNames() { + return new HashSet<>(Arrays.asList(new String[] { "aaa", "sss" })); + } + }; + processor.addWorker(process.getClass().getSimpleName(), process.getClass(), process); + + boolean ret = serviceStateMachine.onSnapshotLoad(new SnapshotReader() { + + @Override + public void close() throws IOException { + + } + + @Override + public String getPath() { + return "Path"; + } + + @Override + public Set listFiles() { + return null; + } + + @Override + public Message getFileMeta(String fileName) { + return new Message() { + @Override + public Message getDefaultInstanceForType() { + return null; + } + + @Override + public boolean isInitialized() { + return false; + } + + @Override + public List findInitializationErrors() { + return null; + } + + @Override + public String getInitializationErrorString() { + return null; + } + + @Override + public Descriptor getDescriptorForType() { + return null; + } + + @Override + public Map getAllFields() { + return null; + } + + @Override + public boolean hasOneof(OneofDescriptor oneofDescriptor) { + return false; + } + + @Override + public FieldDescriptor getOneofFieldDescriptor(OneofDescriptor oneofDescriptor) { + return null; + } + + @Override + public boolean hasField(FieldDescriptor fieldDescriptor) { + return false; + } + + @Override + public Object getField(FieldDescriptor fieldDescriptor) { + return null; + } + + @Override + public int getRepeatedFieldCount(FieldDescriptor fieldDescriptor) { + return 0; + } + + @Override + public Object getRepeatedField(FieldDescriptor fieldDescriptor, int i) { + return null; + } + + @Override + public UnknownFieldSet getUnknownFields() { + return null; + } + + @Override + public void writeTo(CodedOutputStream codedOutputStream) throws IOException { + + } + + @Override + public int getSerializedSize() { + return 0; + } + + @Override + public Parser getParserForType() { + return null; + } + + @Override + public ByteString toByteString() { + return null; + } + + @Override + public byte[] toByteArray() { + return new byte[0]; + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + + } + + @Override + public void writeDelimitedTo(OutputStream outputStream) throws IOException { + + } + + @Override + public boolean equals(Object o) { + return false; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public String toString() { + return null; + } + + @Override + public Builder newBuilderForType() { + return null; + } + + @Override + public Builder toBuilder() { + return null; + } + }; + } + + @Override + public boolean init(Void opts) { + return false; + } + + @Override + public void shutdown() { + + } + + @Override + public SnapshotMeta load() { + return null; + } + + @Override + public String generateURIForCopy() { + return null; + } + }); + + TimeUnit.MILLISECONDS.sleep(1000); + + Assert.assertTrue(ret); + } + + @Test + public void testRemain() throws InterruptedException { + ServiceStateMachine serviceStateMachine = ServiceStateMachine.getInstance(); + serviceStateMachine.setExecutor(new ThreadPoolExecutor(1, 2, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(20))); + + AtomicInteger leaderstart = new AtomicInteger(); + AtomicInteger leaderstop = new AtomicInteger(); + + serviceStateMachine.setFollowerProcessListener(new FollowerProcessListener() { + @Override + public void startProcess(PeerId leader) { + leaderstop.getAndIncrement(); + } + + @Override + public void stopProcess(PeerId leader) { + leaderstop.getAndIncrement(); + } + }); + + serviceStateMachine.setLeaderProcessListener(new LeaderProcessListener() { + @Override + public void startProcess() { + leaderstart.getAndIncrement(); + } + + @Override + public void stopProcess() { + leaderstart.getAndIncrement(); + } + }); + + serviceStateMachine.onLeaderStart(1); + TimeUnit.MILLISECONDS.sleep(500); + Assert.assertEquals(leaderstart.get(), 1); + + serviceStateMachine.onLeaderStop(Status.OK()); + TimeUnit.MILLISECONDS.sleep(500); + Assert.assertTrue(leaderstart.get() == 2); + + serviceStateMachine.onStartFollowing(new LeaderChangeContext(new PeerId(), 1, Status.OK())); + TimeUnit.MILLISECONDS.sleep(500); + Assert.assertTrue(leaderstop.get() == 1); + + serviceStateMachine.onStopFollowing(new LeaderChangeContext(new PeerId(), 1, Status.OK())); + TimeUnit.MILLISECONDS.sleep(500); + Assert.assertTrue(leaderstop.get() == 2); + + } + + public String testMethod(String ss) { + return ss; + } +} \ No newline at end of file diff --git a/server/store/pom.xml b/server/store/pom.xml index 0d6a266d6..b02b5c0ef 100644 --- a/server/store/pom.xml +++ b/server/store/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index ecca9bcdb..9f5cd48b9 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-parent - 5.2.0-SNAPSHOT + 5.4.5 ../pom.xml 4.0.0 @@ -80,4 +80,4 @@ test - + \ No newline at end of file diff --git a/test/src/main/resources/application.properties b/test/src/main/resources/application.properties index 20ce078a6..5be3d5aac 100644 --- a/test/src/main/resources/application.properties +++ b/test/src/main/resources/application.properties @@ -12,12 +12,15 @@ session.server.sessionServerDataCenter=DefaultDataCenter session.server.syncHeartbeat.fixedDelay=30000 session.server.syncExceptionData.fixedDelay=30000 session.server.printTask.fixedDelay=30000 -session.server.schedulerHeartbeatTimeout=30 -session.server.schedulerHeartbeatFirstDelay=30 -session.server.schedulerHeartbeatExpBackOffBound=10 +session.server.schedulerHeartbeatTimeout=3 +session.server.schedulerHeartbeatFirstDelay=3 +session.server.schedulerHeartbeatExpBackOffBound=1 session.server.schedulerFetchDataTimeout=3 session.server.schedulerFetchDataFirstDelay=3 session.server.schedulerFetchDataExpBackOffBound=10 +session.server.renewAndSnapshotSilentPeriodSec=10 +session.server.renewDatumWheelTaskDelaySec=10 +session.server.renewDatumWheelTaskRandomFirstDelaySec=10 ## data node configuration data.server.logging.level=INFO diff --git a/test/src/main/resources/logback-spring.xml b/test/src/main/resources/logback-spring.xml index adbee9009..a1f43c7db 100644 --- a/test/src/main/resources/logback-spring.xml +++ b/test/src/main/resources/logback-spring.xml @@ -4,14 +4,14 @@ - + [%d{ISO8601}][%p][%t][%c{0}] - %m%n - WARN + INFO diff --git a/test/src/test/java/com/alipay/sofa/registry/test/BaseIntegrationTest.java b/test/src/test/java/com/alipay/sofa/registry/test/BaseIntegrationTest.java index 94c48416c..8cd0e59f7 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/BaseIntegrationTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/BaseIntegrationTest.java @@ -27,6 +27,7 @@ import com.alipay.sofa.registry.client.task.AbstractWorkerThread; import com.alipay.sofa.registry.client.task.WorkerThread; import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.sessionserver.CancelAddressRequest; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.URL; @@ -128,7 +129,8 @@ public static void startServerIfNecessary() throws Exception { private static void initRegistryClientAndChannel() { if (registryClient1 == null) { RegistryClientConfig config = DefaultRegistryClientConfigBuilder.start() - .setAppName("testApp1").setDataCenter(LOCAL_DATACENTER).setZone(LOCAL_REGION) + .setSyncConfigRetryInterval(60000).setAppName("testApp1") + .setDataCenter(LOCAL_DATACENTER).setZone(LOCAL_REGION) .setRegistryEndpoint(LOCAL_ADDRESS).setRegistryEndpointPort(sessionPort).build(); registryClient1 = new DefaultRegistryClient(config); registryClient1.init(); @@ -154,6 +156,7 @@ private static void initRegistryClientAndChannel() { public static class MySubscriberDataObserver implements SubscriberDataObserver { @Override public void handleData(String dataId, UserData data) { + BaseIntegrationTest.dataId = dataId; BaseIntegrationTest.userData = data; } @@ -162,8 +165,10 @@ public void handleData(String dataId, UserData data) { protected static void clientOff() throws Exception { startServerIfNecessary(); List connectIds = new ArrayList<>(); - connectIds.add(LOCAL_ADDRESS + ":" + getSourcePort(registryClient1)); - connectIds.add(LOCAL_ADDRESS + ":" + getSourcePort(registryClient2)); + connectIds.add(LOCAL_ADDRESS + ":" + getSourcePort(registryClient1) + + ValueConstants.CONNECT_ID_SPLIT + LOCAL_ADDRESS + ":9600"); + connectIds.add(LOCAL_ADDRESS + ":" + getSourcePort(registryClient2) + + ValueConstants.CONNECT_ID_SPLIT + LOCAL_ADDRESS + ":9600"); CommonResponse response = sessionChannel .getWebTarget() .path("api/clients/off") @@ -182,12 +187,13 @@ protected static void clientOff() throws Exception { } protected static void clearData() throws Exception { - DatumCache.getAll().clear(); + DatumCache datumCache = (DatumCache) dataApplicationContext.getBean("datumCache"); + datumCache.getAll().clear(); List connectIds = new ArrayList<>(Arrays.asList( NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)), NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient2)))); for (String connectId : connectIds) { - Map publisherMap = DatumCache.getByHost(connectId); + Map publisherMap = datumCache.getByConnectId(connectId); if (publisherMap != null) { publisherMap.clear(); } diff --git a/test/src/test/java/com/alipay/sofa/registry/test/pubsub/PubSubTest.java b/test/src/test/java/com/alipay/sofa/registry/test/pubsub/PubSubTest.java index bb7c60a42..ab7495f50 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/pubsub/PubSubTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/pubsub/PubSubTest.java @@ -45,7 +45,7 @@ public void publisherTest() throws InterruptedException { PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); SubscriberRegistration subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); @@ -53,7 +53,7 @@ public void publisherTest() throws InterruptedException { registryClient1.register(subReg); - Thread.sleep(1000L); + Thread.sleep(2000L); assertEquals(dataId, this.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); assertEquals(1, userData.getZoneData().size()); @@ -80,11 +80,11 @@ public void subscriberTest() throws InterruptedException { new MySubscriberDataObserver()); subReg.setScopeEnum(ScopeEnum.dataCenter); registryClient1.register(subReg); - Thread.sleep(500L); + Thread.sleep(2000L); PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(1000L); + Thread.sleep(2000L); assertEquals(dataId, this.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); @@ -112,7 +112,7 @@ public void multiClientTest() throws InterruptedException { PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); SubscriberRegistration subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); @@ -120,7 +120,7 @@ public void multiClientTest() throws InterruptedException { registryClient2.register(subReg); - Thread.sleep(1000L); + Thread.sleep(2000L); assertEquals(dataId, this.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); assertEquals(1, userData.getZoneData().size()); @@ -139,11 +139,11 @@ public void multiClientTest() throws InterruptedException { subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); subReg.setScopeEnum(ScopeEnum.global); registryClient1.register(subReg); - Thread.sleep(500L); + Thread.sleep(2000L); registration = new PublisherRegistration(dataId); registryClient2.register(registration, value); - Thread.sleep(1000L); + Thread.sleep(2000L); assertEquals(dataId, this.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); @@ -163,12 +163,12 @@ public void multiClientTest() throws InterruptedException { subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); subReg.setScopeEnum(ScopeEnum.zone); registryClient1.register(subReg); - Thread.sleep(500L); + Thread.sleep(2000L); registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); registryClient2.register(registration, value); - Thread.sleep(1000L); + Thread.sleep(2000L); assertEquals(dataId, this.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); diff --git a/test/src/test/java/com/alipay/sofa/registry/test/pubsub/TempPublisherTest.java b/test/src/test/java/com/alipay/sofa/registry/test/pubsub/TempPublisherTest.java index 391b3d4d5..e824570d0 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/pubsub/TempPublisherTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/pubsub/TempPublisherTest.java @@ -17,6 +17,8 @@ package com.alipay.sofa.registry.test.pubsub; import com.alipay.sofa.registry.client.api.model.RegistryType; +import com.alipay.sofa.registry.client.api.model.UserData; +import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; import com.alipay.sofa.registry.common.model.PublishType; import com.alipay.sofa.registry.common.model.ServerDataBox; @@ -32,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; @@ -44,18 +47,23 @@ @RunWith(SpringRunner.class) public class TempPublisherTest extends BaseIntegrationTest { @Test - public void doTest() throws Exception { - String dataId = "test-dataId-" + System.currentTimeMillis(); + public synchronized void doTest() throws Exception { + String dataId = "test-dataId-" +this.getClass().getName()+ System.currentTimeMillis(); String value = "test publish temp data"; - + AtomicReference dataIdRef = new AtomicReference<>(); + AtomicReference userDataRef = new AtomicReference<>(); // register SubscriberRegistration SubscriberRegistration subReg = new SubscriberRegistration(dataId, - new MySubscriberDataObserver()); + (dataIdIn, data) -> { + dataIdRef.set(dataIdIn); + userDataRef.set(data); + }); + subReg.setScopeEnum(ScopeEnum.zone); registryClient1.register(subReg); - Thread.sleep(1000L); - assertEquals(dataId, this.dataId); - assertEquals(0, userData.getZoneData().size()); + Thread.sleep(5000L); + assertEquals(dataId, dataIdRef.get()); + assertEquals(0, userDataRef.get().getZoneData().size()); // publish temp data Publisher tempPublisher = new Publisher(); @@ -76,19 +84,74 @@ public void doTest() throws Exception { sessionApplicationContext.getBean(DataNodeService.class).register(tempPublisher); // data size is 1 - Thread.sleep(1000L); - assertEquals(dataId, this.dataId); - assertEquals(1, userData.getZoneData().size()); - userData = null; + Thread.sleep(5000L); + assertEquals(dataId, dataIdRef.get()); + assertEquals(1, userDataRef.get().getZoneData().size()); + + userDataRef.set(null); + dataIdRef.set(null); registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); // register another SubscriberRegistration, data size is 0 - subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); + subReg = new SubscriberRegistration(dataId, + (dataIdIn, data) -> { + dataIdRef.set(dataIdIn); + userDataRef.set(data); + }); subReg.setScopeEnum(ScopeEnum.zone); registryClient1.register(subReg); - Thread.sleep(1000L); - assertEquals(dataId, this.dataId); - assertEquals(0, userData.getZoneData().size()); + Thread.sleep(5000L); + assertEquals(dataId, dataIdRef.get()); + assertEquals(0, userDataRef.get().getZoneData().size()); + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + } + + @Test + public synchronized void doTestPubAndTempPubSameTime() throws Exception { + String dataId = "test-same-time-pub&tempPub-" + System.currentTimeMillis(); + String value = "test same time publish"; + AtomicReference dataIdRef = new AtomicReference<>(); + AtomicReference userDataRef = new AtomicReference<>(); + + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + (dataIdIn, data) -> { + dataIdRef.set(dataIdIn); + userDataRef.set(data); + }); + subReg.setScopeEnum(ScopeEnum.zone); + registryClient1.register(subReg); + Thread.sleep(5000L); + + // publish data + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, "test publish"); + + Thread.sleep(5000L); + + // publish temp data + Publisher tempPublisher = new Publisher(); + tempPublisher.setPublishType(PublishType.TEMPORARY); + tempPublisher.setCell(LOCAL_REGION); + tempPublisher.setDataId(dataId); + tempPublisher.setGroup(DEFAULT_GROUP); + tempPublisher.setInstanceId(DEFAULT_INSTANCE_ID); + tempPublisher.setVersion(System.currentTimeMillis()); + tempPublisher.setRegisterTimestamp(System.currentTimeMillis()); + tempPublisher.setClientRegisterTimestamp(System.currentTimeMillis()); + tempPublisher.setRegisterId(UUID.randomUUID().toString()); + tempPublisher.setDataInfoId(DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, + DEFAULT_GROUP)); + List dataBoxData = new ArrayList(1); + dataBoxData.add(new ServerDataBox(object2bytes(value))); + tempPublisher.setDataList(dataBoxData); + sessionApplicationContext.getBean(DataNodeService.class).register(tempPublisher); + + Thread.sleep(5000L); + + assertEquals(1, userDataRef.get().getZoneData().size()); + assertEquals(2, userDataRef.get().getZoneData().get(LOCAL_REGION).size()); + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.PUBLISHER); } } diff --git a/test/src/test/java/com/alipay/sofa/registry/test/renew/RenewTest.java b/test/src/test/java/com/alipay/sofa/registry/test/renew/RenewTest.java new file mode 100644 index 000000000..c531420c4 --- /dev/null +++ b/test/src/test/java/com/alipay/sofa/registry/test/renew/RenewTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.test.renew; + +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; +import static org.junit.Assert.assertEquals; + +import javax.ws.rs.core.MediaType; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + +import com.alipay.sofa.registry.client.api.model.RegistryType; +import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; +import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.core.model.Result; +import com.alipay.sofa.registry.core.model.ScopeEnum; +import com.alipay.sofa.registry.server.data.cache.DatumCache; +import com.alipay.sofa.registry.test.BaseIntegrationTest; + +/** + * @author kezhu.wukz + */ +@RunWith(SpringRunner.class) +public class RenewTest extends BaseIntegrationTest { + + /** + * clean datum -> renew + */ + @Test + public void testRenewAfterClean() throws InterruptedException { + String dataId = "test-dataId-" + System.currentTimeMillis(); + String value = "test publish"; + + // pub/sub + { + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + + registryClient1.register(subReg); + } + + Thread.sleep(5000L); + + // check sub + { + assertEquals(dataId, this.dataId); + assertEquals(LOCAL_REGION, userData.getLocalZone()); + assertEquals(1, userData.getZoneData().size()); + assertEquals(1, userData.getZoneData().values().size()); + assertEquals(true, userData.getZoneData().containsKey(LOCAL_REGION)); + assertEquals(1, userData.getZoneData().get(LOCAL_REGION).size()); + assertEquals(value, userData.getZoneData().get(LOCAL_REGION).get(0)); + } + + //clean datum and unsub + { + //pub again to ignore renew for 10sec + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + //un sub, because will new sub below + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + // sleep to sure pub is done + Thread.sleep(2000L); + // clean all datum + DatumCache datumCache = (DatumCache) dataApplicationContext.getBean("datumCache"); + datumCache.cleanDatum(LOCAL_DATACENTER, + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)); + } + + // new sub, and got empty datum + { + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + registryClient1.register(subReg); + } + Thread.sleep(3000L); + assertEquals(0, userData.getZoneData().size()); + + //renew in 20sec + Thread.sleep(20000L); + assertEquals(1, userData.getZoneData().size()); + + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.PUBLISHER); + } + + /** + * close renew -> clean datum + */ + @Test + public void testNoRenewAfterClean() throws InterruptedException { + // close renew + Assert.assertTrue(metaChannel.getWebTarget().path("renewSwitch/disable") + .request(MediaType.APPLICATION_JSON).get(Result.class).isSuccess()); + Thread.sleep(2000L); + + String dataId = "test-dataId-" + System.currentTimeMillis(); + String value = "test publish"; + + // pub/sub + { + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + + registryClient1.register(subReg); + } + + Thread.sleep(5000L); + + // check sub + { + assertEquals(dataId, this.dataId); + assertEquals(LOCAL_REGION, userData.getLocalZone()); + assertEquals(1, userData.getZoneData().size()); + assertEquals(1, userData.getZoneData().values().size()); + assertEquals(true, userData.getZoneData().containsKey(LOCAL_REGION)); + assertEquals(1, userData.getZoneData().get(LOCAL_REGION).size()); + assertEquals(value, userData.getZoneData().get(LOCAL_REGION).get(0)); + } + + //clean datum and unsub + { + //pub again to ignore renew for 10sec + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + //un sub, because will new sub below + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + // sleep to sure pub is done + Thread.sleep(2000L); + // clean all datum + DatumCache datumCache = (DatumCache) dataApplicationContext.getBean("datumCache"); + datumCache.cleanDatum(LOCAL_DATACENTER, + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)); + } + + // new sub, and got empty datum + { + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + registryClient1.register(subReg); + } + Thread.sleep(3000L); + assertEquals(0, userData.getZoneData().size()); + + //renew is disable, so it's also 0 in 20sec + Thread.sleep(20000L); + assertEquals(0, userData.getZoneData().size()); + + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); + registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.PUBLISHER); + + // open renew + Assert.assertTrue(metaChannel.getWebTarget().path("renewSwitch/enable") + .request(MediaType.APPLICATION_JSON).get(Result.class).isSuccess()); + Thread.sleep(2000L); + } + +} \ No newline at end of file diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/data/DataDigestResourceTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/data/DataDigestResourceTest.java index 2db2e7361..902085d80 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/resource/data/DataDigestResourceTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/data/DataDigestResourceTest.java @@ -16,6 +16,28 @@ */ package com.alipay.sofa.registry.test.resource.data; +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.MediaType; + +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.net.NetUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + import com.alipay.sofa.registry.client.api.model.RegistryType; import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; @@ -23,23 +45,6 @@ import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.core.model.ScopeEnum; import com.alipay.sofa.registry.test.BaseIntegrationTest; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; - -import javax.ws.rs.client.Entity; -import javax.ws.rs.core.GenericType; -import javax.ws.rs.core.MediaType; -import java.util.HashMap; -import java.util.Map; - -import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; -import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author xuanbei @@ -53,16 +58,17 @@ public static void beforeClass() throws Exception { clientOff(); dataId = "test-dataId-" + System.currentTimeMillis(); value = "DataDigestResourceTest"; + Thread.sleep(2000L); PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); SubscriberRegistration subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); subReg.setScopeEnum(ScopeEnum.dataCenter); registryClient1.register(subReg); - Thread.sleep(500L); + Thread.sleep(2000L); } @AfterClass @@ -89,7 +95,8 @@ public void testGetDatumByDataInfoId() throws Exception { @Test public void testGetPublishersByConnectId() throws Exception { Map parameters = new HashMap<>(); - parameters.put(LOCAL_ADDRESS, String.valueOf(getSourcePort(registryClient1))); + parameters.put(NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)), + NetUtil.genHost(LOCAL_ADDRESS, 9600)); Map> publisherMap = dataChannel .getWebTarget() .path("digest/connect/query") @@ -98,17 +105,28 @@ public void testGetPublishersByConnectId() throws Exception { new GenericType>>() { }); assertEquals(1, publisherMap.size()); - assertEquals(1, - publisherMap.get(LOCAL_ADDRESS + ":" + String.valueOf(getSourcePort(registryClient1))) + assertEquals( + 1, + publisherMap.get( + NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)) + + ValueConstants.CONNECT_ID_SPLIT + NetUtil.genHost(LOCAL_ADDRESS, 9600)) .size()); - assertEquals(dataId, - publisherMap.get(LOCAL_ADDRESS + ":" + String.valueOf(getSourcePort(registryClient1))) - .values().iterator().next().getDataId()); + assertEquals( + dataId, + publisherMap + .get( + NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)) + + ValueConstants.CONNECT_ID_SPLIT + + NetUtil.genHost(LOCAL_ADDRESS, 9600)).values().iterator().next() + .getDataId()); assertEquals( value, bytes2Object(publisherMap - .get(LOCAL_ADDRESS + ":" + String.valueOf(getSourcePort(registryClient1))).values() - .iterator().next().getDataList().get(0).getBytes())); + .get( + NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)) + + ValueConstants.CONNECT_ID_SPLIT + + NetUtil.genHost(LOCAL_ADDRESS, 9600)).values().iterator().next() + .getDataList().get(0).getBytes())); } @Test @@ -117,4 +135,29 @@ public void testGetDatumCount() { .request(APPLICATION_JSON).get(String.class); assertTrue(countResult.contains("[Publisher] size of publisher in DefaultDataCenter is 1")); } + + @Test + public void testGetServerListAll() throws Exception { + + Map> sessionMap = dataChannel.getWebTarget() + .path("digest/session/serverList/query").request(APPLICATION_JSON) + .get(new GenericType>>() { + }); + assertEquals(1, sessionMap.size()); + assertEquals(1, sessionMap.get(LOCAL_DATACENTER).size()); + assertTrue(sessionMap.get(LOCAL_DATACENTER).get(0).contains(LOCAL_ADDRESS)); + + Map> metaMap = dataChannel.getWebTarget() + .path("digest/meta/serverList/query").request(APPLICATION_JSON) + .get(new GenericType>>() { + }); + assertEquals(metaMap.get(LOCAL_DATACENTER).get(0), LOCAL_ADDRESS); + + Map> dataMap = dataChannel.getWebTarget() + .path("digest/data/serverList/query").request(APPLICATION_JSON) + .get(new GenericType>>() { + }); + assertEquals(1, dataMap.size()); + assertEquals(0, dataMap.get(LOCAL_DATACENTER).size()); + } } \ No newline at end of file diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/BlacklistTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/BlacklistTest.java new file mode 100644 index 000000000..fff295990 --- /dev/null +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/BlacklistTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.test.resource.meta; + +import com.alipay.sofa.registry.client.api.model.RegistryType; +import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; +import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.Subscriber; +import com.alipay.sofa.registry.core.model.Result; +import com.alipay.sofa.registry.core.model.ScopeEnum; +import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.server.session.filter.blacklist.BlacklistConstants; +import com.alipay.sofa.registry.test.BaseIntegrationTest; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.MediaType; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author shangyu.wh + * @since 2019/1/14 + */ +@RunWith(SpringRunner.class) +public class BlacklistTest extends BaseIntegrationTest { + @Test + public void testBlacklistUpdate() throws Exception { + + Map>> map = new HashMap<>(); + Set set1 = new HashSet<>(); + Set set2 = new HashSet<>(); + set1.add("193.165.0.1"); + set1.add("193.165.0.2"); + set2.add("193.165.0.3"); + set2.add("193.165.0.4"); + + Map> map1 = Maps.newHashMap(); + map1.put(BlacklistConstants.IP_FULL, set1); + + Map> map2 = Maps.newHashMap(); + map2.put(BlacklistConstants.IP_FULL, set2); + + map.put(BlacklistConstants.FORBIDDEN_PUB, map1); + map.put(BlacklistConstants.FORBIDDEN_SUB_BY_PREFIX, map2); + + ObjectMapper mapper = new ObjectMapper(); + + Result response = metaChannel + .getWebTarget() + .path("blacklist/update") + .request() + .post(Entity.entity(mapper.writeValueAsString(map), MediaType.APPLICATION_JSON), + Result.class); + + assertTrue(response.isSuccess()); + + } + + @Test + public void testBlacklistUpdatePub() throws Exception { + + Map>> map = new HashMap<>(); + Set set1 = new HashSet<>(); + Set set2 = new HashSet<>(); + String local = NetUtil.getLocalAddress().getHostAddress(); + set1.add(local); + set2.add(local); + + Map> map1 = Maps.newHashMap(); + map1.put(BlacklistConstants.IP_FULL, set1); + + Map> map2 = Maps.newHashMap(); + map2.put(BlacklistConstants.IP_FULL, set2); + + map.put(BlacklistConstants.FORBIDDEN_PUB, map1); + + ObjectMapper mapper = new ObjectMapper(); + + Result response = metaChannel + .getWebTarget() + .path("blacklist/update") + .request() + .post(Entity.entity(mapper.writeValueAsString(map), MediaType.APPLICATION_JSON), + Result.class); + + assertTrue(response.isSuccess()); + + BaseIntegrationTest.dataId = null; + + //wait for new list meta dispatch to session + Thread.sleep(2000L); + + String dataId = "test-dataId-blacklist"; + String value = "test blacklist"; + + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + Thread.sleep(2000L); + + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + + registryClient1.register(subReg); + + Thread.sleep(2000L); + assertEquals(dataId, this.dataId); + assertEquals(LOCAL_REGION, userData.getLocalZone()); + + Map> publisherMap = sessionChannel + .getWebTarget() + .path("digest/pub/data/query") + .queryParam("dataInfoId", + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)) + .request(APPLICATION_JSON).get(new GenericType>>() { + }); + assertEquals(0, publisherMap.size()); + + Map> subscriberMap = sessionChannel + .getWebTarget() + .path("digest/sub/data/query") + .queryParam("dataInfoId", + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)) + .request(APPLICATION_JSON).get(new GenericType>>() { + }); + assertEquals(1, subscriberMap.size()); + assertEquals(1, subscriberMap.get("SUB").size()); + assertEquals(dataId, subscriberMap.get("SUB").get(0).getDataId()); + + } + + @Test + public void testBlacklistUpdateSub() throws Exception { + + Map>> map = new HashMap<>(); + Set set1 = new HashSet<>(); + Set set2 = new HashSet<>(); + String local = NetUtil.getLocalAddress().getHostAddress(); + set1.add(local); + set2.add(local); + + Map> map1 = Maps.newHashMap(); + map1.put(BlacklistConstants.IP_FULL, set1); + + Map> map2 = Maps.newHashMap(); + map2.put(BlacklistConstants.IP_FULL, set2); + + map.put(BlacklistConstants.FORBIDDEN_SUB_BY_PREFIX, map2); + + ObjectMapper mapper = new ObjectMapper(); + + Result response = metaChannel + .getWebTarget() + .path("blacklist/update") + .request() + .post(Entity.entity(mapper.writeValueAsString(map), MediaType.APPLICATION_JSON), + Result.class); + + assertTrue(response.isSuccess()); + + BaseIntegrationTest.dataId = null; + //wait for new list meta dispatch to session + Thread.sleep(2000L); + + String dataId = "test-dataId-blacklist2"; + String value = "test blacklist2"; + + PublisherRegistration registration = new PublisherRegistration(dataId); + registryClient1.register(registration, value); + Thread.sleep(2000L); + + SubscriberRegistration subReg = new SubscriberRegistration(dataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + + registryClient1.register(subReg); + + Thread.sleep(2000L); + assertEquals(dataId, this.dataId); + assertEquals(LOCAL_REGION, userData.getLocalZone()); + + Map> publisherMap = sessionChannel + .getWebTarget() + .path("digest/pub/data/query") + .queryParam("dataInfoId", + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)) + .request(APPLICATION_JSON).get(new GenericType>>() { + }); + assertEquals(1, publisherMap.size()); + assertEquals(1, publisherMap.get("PUB").size()); + assertEquals(dataId, publisherMap.get("PUB").get(0).getDataId()); + assertEquals(value, bytes2Object(publisherMap.get("PUB").get(0).getDataList().get(0) + .getBytes())); + + Map> subscriberMap = sessionChannel + .getWebTarget() + .path("digest/sub/data/query") + .queryParam("dataInfoId", + DataInfo.toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)) + .request(APPLICATION_JSON).get(new GenericType>>() { + }); + assertEquals(0, subscriberMap.size()); + + } + + @AfterClass + public static void afterClass() { + registryClient1.unregister("test-dataId-blacklist", DEFAULT_GROUP, RegistryType.SUBSCRIBER); + registryClient1.unregister("test-dataId-blacklist", DEFAULT_GROUP, RegistryType.PUBLISHER); + registryClient1 + .unregister("test-dataId-blacklist2", DEFAULT_GROUP, RegistryType.SUBSCRIBER); + registryClient1.unregister("test-dataId-blacklist2", DEFAULT_GROUP, RegistryType.PUBLISHER); + } +} diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/StopPushDataSwitchTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/StopPushDataSwitchTest.java index 61bff0670..ca7ff690f 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/StopPushDataSwitchTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/meta/StopPushDataSwitchTest.java @@ -16,7 +16,24 @@ */ package com.alipay.sofa.registry.test.resource.meta; +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.CollectionUtils; + import com.alipay.sofa.registry.client.api.model.RegistryType; +import com.alipay.sofa.registry.client.api.model.UserData; import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; import com.alipay.sofa.registry.common.model.store.Subscriber; @@ -29,18 +46,6 @@ import com.alipay.sofa.registry.task.listener.TaskEvent; import com.alipay.sofa.registry.task.listener.TaskListenerManager; import com.alipay.sofa.registry.test.BaseIntegrationTest; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.util.CollectionUtils; - -import java.util.Map; - -import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; -import static javax.ws.rs.core.MediaType.APPLICATION_JSON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; /** * @author xuanbei @@ -48,40 +53,58 @@ */ @RunWith(SpringRunner.class) public class StopPushDataSwitchTest extends BaseIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(StopPushDataSwitchTest.class); + @Test public void testStopPushDataSwitch() throws Exception { // open stop push switch - assertTrue(metaChannel.getWebTarget().path("stopPushDataSwitch/open") - .request(APPLICATION_JSON).get(Result.class).isSuccess()); - BaseIntegrationTest.dataId = null; + assertTrue( + metaChannel.getWebTarget().path("stopPushDataSwitch/open").request(APPLICATION_JSON).get(Result.class) + .isSuccess()); + Thread.sleep(2000L); + + AtomicReference dataIdResult = new AtomicReference<>(); + AtomicReference userDataResult = new AtomicReference<>(); // register Publisher & Subscriber, Subscriber get no data String dataId = "test-dataId-" + System.currentTimeMillis(); String value = "test stop publish data switch"; + + LOGGER.info("dataidIn:" + dataId); + + SessionServerConfig sessionServerConfig = sessionApplicationContext.getBean(SessionServerConfig.class); + + LOGGER.info("sessionServerConfig.isStopPushSwitch:" + sessionServerConfig.isStopPushSwitch()); + PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); - SubscriberRegistration subReg = new SubscriberRegistration(dataId, - new MySubscriberDataObserver()); + SubscriberRegistration subReg = new SubscriberRegistration(dataId, (dataIdOb, data) -> { + LOGGER.info("sub:" + data); + dataIdResult.set(dataIdOb); + userDataResult.set(data); + }); subReg.setScopeEnum(ScopeEnum.dataCenter); registryClient1.register(subReg); - Thread.sleep(1000L); - assertNull(BaseIntegrationTest.dataId); + Thread.sleep(2000L); + assertNull(dataIdResult.get()); // close stop push switch - assertTrue(metaChannel.getWebTarget().path("stopPushDataSwitch/close") - .request(APPLICATION_JSON).get(Result.class).isSuccess()); - Thread.sleep(1000L); + assertTrue( + metaChannel.getWebTarget().path("stopPushDataSwitch/close").request(APPLICATION_JSON).get(Result.class) + .isSuccess()); + Thread.sleep(2000L); // Subscriber get data, test data - assertEquals(dataId, this.dataId); - assertEquals(LOCAL_REGION, userData.getLocalZone()); - assertEquals(1, userData.getZoneData().size()); - assertEquals(1, userData.getZoneData().values().size()); - assertEquals(true, userData.getZoneData().containsKey(LOCAL_REGION)); - assertEquals(1, userData.getZoneData().get(LOCAL_REGION).size()); - assertEquals(value, userData.getZoneData().get(LOCAL_REGION).get(0)); + assertEquals(dataId, dataIdResult.get()); + assertEquals(LOCAL_REGION, userDataResult.get().getLocalZone()); + assertEquals(1, userDataResult.get().getZoneData().size()); + assertEquals(1, userDataResult.get().getZoneData().values().size()); + assertEquals(true, userDataResult.get().getZoneData().containsKey(LOCAL_REGION)); + assertEquals(1, userDataResult.get().getZoneData().get(LOCAL_REGION).size()); + assertEquals(value, userDataResult.get().getZoneData().get(LOCAL_REGION).get(0)); // unregister Publisher & Subscriber registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); @@ -95,28 +118,41 @@ public void testStopPushDataSwitch() throws Exception { @Test public void testStopPushDataSwitchByCode() throws Exception { // open stop push switch - assertTrue(metaChannel.getWebTarget().path("stopPushDataSwitch/open").request(APPLICATION_JSON) - .get(Result.class) - .isSuccess()); - BaseIntegrationTest.dataId = null; + assertTrue( + metaChannel.getWebTarget().path("stopPushDataSwitch/open").request(APPLICATION_JSON).get(Result.class) + .isSuccess()); + Thread.sleep(2000L); + + AtomicReference dataIdResult = new AtomicReference<>(); + AtomicReference userDataResult = new AtomicReference<>(); // register Publisher & Subscriber, Subscriber get no data String dataId = "test-dataId-hahhahahahha-" + System.currentTimeMillis(); String value = "test stop publish data switch by code"; + + LOGGER.info("dataidIn2:" + dataId); + + SessionServerConfig sessionServerConfig = sessionApplicationContext.getBean(SessionServerConfig.class); + + LOGGER.info("sessionServerConfig.isStopPushSwitch2:" + sessionServerConfig.isStopPushSwitch()); PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); - SubscriberRegistration subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); + SubscriberRegistration subReg = new SubscriberRegistration(dataId, (dataIdOb, data) -> { + LOGGER.info("sub:" + data); + dataIdResult.set(dataIdOb); + userDataResult.set(data); + }); subReg.setScopeEnum(ScopeEnum.dataCenter); registryClient1.register(subReg); - Thread.sleep(1000L); - assertNull(BaseIntegrationTest.dataId); + Thread.sleep(2000L); + assertNull(dataIdResult.get()); // invoke code directly Interests sessionInterests = sessionApplicationContext.getBean(Interests.class); TaskListenerManager taskListenerManager = sessionApplicationContext.getBean(TaskListenerManager.class); - SessionServerConfig sessionServerConfig = sessionApplicationContext.getBean(SessionServerConfig.class); + if (sessionInterests instanceof ReSubscribers) { ReSubscribers reSubscriber = (ReSubscribers) sessionInterests; @@ -138,19 +174,19 @@ public void testStopPushDataSwitchByCode() throws Exception { Thread.sleep(1000); // Subscriber get data, test data - assertEquals(dataId, this.dataId); - assertEquals(LOCAL_REGION, userData.getLocalZone()); - assertEquals(1, userData.getZoneData().size()); - assertEquals(1, userData.getZoneData().values().size()); - assertEquals(true, userData.getZoneData().containsKey(LOCAL_REGION)); - assertEquals(1, userData.getZoneData().get(LOCAL_REGION).size()); - assertEquals(value, userData.getZoneData().get(LOCAL_REGION).get(0)); + assertEquals(dataId, dataIdResult.get()); + assertEquals(LOCAL_REGION, userDataResult.get().getLocalZone()); + assertEquals(1, userDataResult.get().getZoneData().size()); + assertEquals(1, userDataResult.get().getZoneData().values().size()); + assertEquals(true, userDataResult.get().getZoneData().containsKey(LOCAL_REGION)); + assertEquals(1, userDataResult.get().getZoneData().get(LOCAL_REGION).size()); + assertEquals(value, userDataResult.get().getZoneData().get(LOCAL_REGION).get(0)); // unregister Publisher & Subscriber, close stop push switch registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.SUBSCRIBER); registryClient1.unregister(dataId, DEFAULT_GROUP, RegistryType.PUBLISHER); - assertTrue(metaChannel.getWebTarget().path("stopPushDataSwitch/close").request(APPLICATION_JSON) - .get(Result.class) - .isSuccess()); + assertTrue( + metaChannel.getWebTarget().path("stopPushDataSwitch/close").request(APPLICATION_JSON).get(Result.class) + .isSuccess()); } } diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/session/ClientsOpenResourceTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/ClientsOpenResourceTest.java index f456d5b01..c9a74282e 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/resource/session/ClientsOpenResourceTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/ClientsOpenResourceTest.java @@ -38,7 +38,7 @@ public void testClientOff() throws Exception { String value = "test client off"; PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(1000L); + Thread.sleep(2000L); String countResult = dataChannel.getWebTarget().path("digest/datum/count") .request(APPLICATION_JSON).get(String.class); diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/session/SessionDigestResourceTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/SessionDigestResourceTest.java index c9107a5dc..f76584406 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/resource/session/SessionDigestResourceTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/SessionDigestResourceTest.java @@ -19,6 +19,7 @@ import com.alipay.sofa.registry.client.api.model.RegistryType; import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; import com.alipay.sofa.registry.common.model.store.DataInfo; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.common.model.store.Subscriber; @@ -37,11 +38,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author xuanbei @@ -57,13 +60,13 @@ public static void beforeClass() throws Exception { PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); SubscriberRegistration subReg = new SubscriberRegistration(dataId, new MySubscriberDataObserver()); subReg.setScopeEnum(ScopeEnum.dataCenter); registryClient1.register(subReg); - Thread.sleep(500L); + Thread.sleep(2000L); } @AfterClass @@ -116,7 +119,8 @@ public void testGetSessionDataByDataInfoId() throws Exception { @Test public void testetSessionDataByConnectId() throws Exception { List connectIds = new ArrayList<>(); - connectIds.add(NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1))); + connectIds.add(NetUtil.genHost(LOCAL_ADDRESS, getSourcePort(registryClient1)) + + ValueConstants.CONNECT_ID_SPLIT + NetUtil.genHost(LOCAL_ADDRESS, 9600)); Map> publisherMap = sessionChannel .getWebTarget() .path("digest/pub/connect/query") @@ -167,4 +171,20 @@ public void testGetPushSwitch() { assertEquals(1, result.size()); assertEquals("open", result.get("pushSwitch")); } + + @Test + public void testGetDataInfoIdList() { + Set result = sessionChannel.getWebTarget().path("digest/getDataInfoIdList") + .request(APPLICATION_JSON).get(Set.class); + + assertTrue(result.contains(DataInfo + .toDataInfoId(dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP))); + } + + @Test + public void testCheckSumDataInfoIdList() { + int result = sessionChannel.getWebTarget().path("digest/checkSumDataInfoIdList") + .request(APPLICATION_JSON).get(int.class); + assertTrue(result != 0); + } } diff --git a/test/src/test/java/com/alipay/sofa/registry/test/resource/session/remoting/DataNodeExchangerTest.java b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/remoting/DataNodeExchangerTest.java new file mode 100644 index 000000000..374a8f092 --- /dev/null +++ b/test/src/test/java/com/alipay/sofa/registry/test/resource/session/remoting/DataNodeExchangerTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.test.resource.session.remoting; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + +import com.alipay.sofa.registry.common.model.dataserver.GetDataVersionRequest; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.remoting.exchange.RequestException; +import com.alipay.sofa.registry.remoting.exchange.message.Request; +import com.alipay.sofa.registry.server.session.remoting.DataNodeExchanger; +import com.alipay.sofa.registry.test.BaseIntegrationTest; + +/** + * @author kezhu.wukz + * @since 2020/03/03 + */ +@RunWith(SpringRunner.class) +public class DataNodeExchangerTest extends BaseIntegrationTest { + + @Test + public void getTaskClosureTest() { + Request request = new Request() { + @Override + public GetDataVersionRequest getRequestBody() { + GetDataVersionRequest getDataVersionRequest = new GetDataVersionRequest(); + return getDataVersionRequest; + } + + @Override + public URL getRequestUrl() { + return new URL("127.0.0.1", 54321); + } + }; + DataNodeExchanger dataNodeExchanger = (DataNodeExchanger) sessionApplicationContext + .getBean("dataNodeExchanger"); + + try { + dataNodeExchanger.request(request); + assert false; + } catch (RequestException e) { + } + + } +} diff --git a/test/src/test/java/com/alipay/sofa/registry/test/sync/DataSyncTest.java b/test/src/test/java/com/alipay/sofa/registry/test/sync/DataSyncTest.java index 2dcbb7416..93ef9a882 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/sync/DataSyncTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/sync/DataSyncTest.java @@ -16,7 +16,16 @@ */ package com.alipay.sofa.registry.test.sync; -import com.alipay.remoting.Connection; +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; import com.alipay.sofa.registry.common.model.CommonResponse; import com.alipay.sofa.registry.common.model.dataserver.NotifyDataSyncRequest; @@ -33,15 +42,6 @@ import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerConnectionFactory; import com.alipay.sofa.registry.server.data.remoting.dataserver.handler.DataSyncServerConnectionHandler; import com.alipay.sofa.registry.test.BaseIntegrationTest; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.test.context.junit4.SpringRunner; - -import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; -import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author xuanbei @@ -53,6 +53,7 @@ public class DataSyncTest extends BaseIntegrationTest { private static DataServerConnectionFactory dataServerConnectionFactory; private static Server dataSyncServer; private static String remoteIP; + private static BoltChannel boltChannel; @BeforeClass public static void beforeClass() throws Exception { @@ -67,20 +68,24 @@ public static void beforeClass() throws Exception { dataSyncServer = boltExchange.open(new URL(NetUtil.getLocalAddress().getHostAddress(), TEST_SYNC_PORT), new ChannelHandler[] { new MockSyncDataHandler(), dataApplicationContext.getBean(DataSyncServerConnectionHandler.class) }); - remoteIP = ((BoltChannel) dataNodeExchanger.connect(new URL(LOCAL_ADDRESS, TEST_SYNC_PORT))) - .getConnection().getLocalIP(); + boltChannel = (BoltChannel) dataNodeExchanger + .connect(new URL(LOCAL_ADDRESS, TEST_SYNC_PORT)); + System.out.println("testsyncserver remote connect remote:" + + boltChannel.getConnection().getRemoteAddress() + " local:" + + boltChannel.getConnection().getLocalAddress()); + remoteIP = boltChannel.getConnection().getLocalIP(); Thread.sleep(500); } @Test public void doTest() throws Exception { // post sync data request - Connection connection = dataServerConnectionFactory.getConnection(remoteIP); NotifyDataSyncRequest request = new NotifyDataSyncRequest(DataInfo.toDataInfoId( MockSyncDataHandler.dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP), LOCAL_DATACENTER, MockSyncDataHandler.version, DataSourceTypeEnum.SYNC.toString()); - CommonResponse commonResponse = (CommonResponse) dataSyncServer.sendSync( - dataSyncServer.getChannel(connection.getRemoteAddress()), request, 1000); + CommonResponse commonResponse = (CommonResponse) dataSyncServer.sendSync(dataSyncServer + .getChannel(new URL(remoteIP, boltChannel.getConnection().getLocalPort())), request, + 1000); assertTrue(commonResponse.isSuccess()); // register Subscriber @@ -90,7 +95,7 @@ public void doTest() throws Exception { registryClient1.register(subReg); // assert result - Thread.sleep(1000L); + Thread.sleep(5000L); assertEquals(MockSyncDataHandler.dataId, BaseIntegrationTest.dataId); assertEquals(LOCAL_REGION, userData.getLocalZone()); assertEquals(1, userData.getZoneData().size()); diff --git a/test/src/test/java/com/alipay/sofa/registry/test/sync/MockSyncDataHandler.java b/test/src/test/java/com/alipay/sofa/registry/test/sync/MockSyncDataHandler.java index 502fc2e30..1abb34d29 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/sync/MockSyncDataHandler.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/sync/MockSyncDataHandler.java @@ -16,6 +16,18 @@ */ package com.alipay.sofa.registry.test.sync; +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; +import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_ADDRESS; +import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_DATACENTER; +import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_REGION; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + import com.alipay.sofa.registry.common.model.GenericResponse; import com.alipay.sofa.registry.common.model.Node; import com.alipay.sofa.registry.common.model.ServerDataBox; @@ -28,20 +40,9 @@ import com.alipay.sofa.registry.common.model.store.URL; import com.alipay.sofa.registry.remoting.Channel; import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; +import com.alipay.sofa.registry.util.DatumVersionUtil; import com.alipay.sofa.registry.util.ParaCheckUtil; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; -import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; -import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_ADDRESS; -import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_DATACENTER; -import static com.alipay.sofa.registry.test.BaseIntegrationTest.LOCAL_REGION; - /** * mock SyncDataHandler, it will return fixed Datum. * @@ -51,7 +52,7 @@ public class MockSyncDataHandler extends AbstractServerHandler { public static String dataId = "test-dataId-" + System.currentTimeMillis(); public static String value = "MockSyncDataHandler"; - public static long version = System.currentTimeMillis(); + public static long version = DatumVersionUtil.nextId(); @Override public void checkParam(SyncDataRequest request) throws RuntimeException { @@ -97,6 +98,7 @@ private Publisher createPublisher() { publisher.setInstanceId(DEFAULT_INSTANCE_ID); publisher.setVersion(version); publisher.setSourceAddress(new URL(LOCAL_ADDRESS, 65343)); + publisher.setTargetAddress(new URL(LOCAL_ADDRESS, 9600)); publisher.setDataInfoId(DataInfo.toDataInfoId(MockSyncDataHandler.dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP)); return publisher; diff --git a/test/src/test/java/com/alipay/sofa/registry/test/sync/SessionNotifyTest.java b/test/src/test/java/com/alipay/sofa/registry/test/sync/SessionNotifyTest.java new file mode 100644 index 000000000..4349a808e --- /dev/null +++ b/test/src/test/java/com/alipay/sofa/registry/test/sync/SessionNotifyTest.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.test.sync; + +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_DATA_CENTER; +import static com.alipay.sofa.registry.client.constants.ValueConstants.DEFAULT_GROUP; +import static com.alipay.sofa.registry.common.model.constants.ValueConstants.DEFAULT_INSTANCE_ID; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + +import com.alipay.remoting.Connection; +import com.alipay.sofa.registry.common.model.CommonResponse; +import com.alipay.sofa.registry.common.model.GenericResponse; +import com.alipay.sofa.registry.common.model.Node; +import com.alipay.sofa.registry.common.model.ServerDataBox; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.dataserver.GetDataRequest; +import com.alipay.sofa.registry.common.model.sessionserver.DataChangeRequest; +import com.alipay.sofa.registry.common.model.store.BaseInfo.ClientVersion; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.common.model.store.Publisher; +import com.alipay.sofa.registry.common.model.store.URL; +import com.alipay.sofa.registry.net.NetUtil; +import com.alipay.sofa.registry.remoting.CallbackHandler; +import com.alipay.sofa.registry.remoting.Channel; +import com.alipay.sofa.registry.remoting.ChannelHandler; +import com.alipay.sofa.registry.remoting.Client; +import com.alipay.sofa.registry.remoting.Server; +import com.alipay.sofa.registry.remoting.bolt.BoltChannel; +import com.alipay.sofa.registry.remoting.bolt.BoltClient; +import com.alipay.sofa.registry.remoting.bolt.exchange.BoltExchange; +import com.alipay.sofa.registry.remoting.exchange.Exchange; +import com.alipay.sofa.registry.server.data.remoting.dataserver.DataServerConnectionFactory; +import com.alipay.sofa.registry.server.data.remoting.dataserver.handler.DataSyncServerConnectionHandler; +import com.alipay.sofa.registry.server.data.remoting.handler.AbstractServerHandler; +import com.alipay.sofa.registry.server.session.cache.DatumKey; +import com.alipay.sofa.registry.server.session.cache.Key; +import com.alipay.sofa.registry.server.session.cache.Key.KeyType; +import com.alipay.sofa.registry.server.session.cache.SessionCacheService; +import com.alipay.sofa.registry.server.session.store.SessionInterests; +import com.alipay.sofa.registry.test.BaseIntegrationTest; +import com.alipay.sofa.registry.util.DatumVersionUtil; +import com.alipay.sofa.registry.util.NamedThreadFactory; +import com.alipay.sofa.registry.util.ParaCheckUtil; + +/** + * @author xuanbei + * @since 2019/1/16 + */ +@RunWith(SpringRunner.class) +public class SessionNotifyTest extends BaseIntegrationTest { + private static final int TEST_SYNC_PORT = 9677; + private static DataServerConnectionFactory dataServerConnectionFactory; + private static Server dataSyncServer; + private static String remoteIP; + private static BoltChannel boltChannel; + + private static Map boltChannelMap = new ConcurrentHashMap<>(); + + private static Map boltChannelMapInt = new ConcurrentHashMap<>(); + + private static final int connectNum = 20; + + private static final int threadNum = 20; + + private static final int idNum = 100; + + private static SessionCacheService sessionCacheService; + + private static SessionInterests sessionInterests; + + private static Datum datum; + + private static BoltClient boltClientFetch = new BoltClient(1); + + private static ThreadPoolExecutor executor = new ThreadPoolExecutor(100, 100, + 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + new NamedThreadFactory( + "TestSessionNotifyFetch")); + + @BeforeClass + public static void beforeClass() throws Exception { + startServerIfNecessary(); + BoltExchange boltExchange = (BoltExchange) dataApplicationContext.getBean("boltExchange"); + dataServerConnectionFactory = dataApplicationContext.getBean("dataServerConnectionFactory", + DataServerConnectionFactory.class); + + sessionCacheService = sessionApplicationContext.getBean("sessionCacheService", + SessionCacheService.class); + + sessionInterests = sessionApplicationContext.getBean("sessionInterests", + SessionInterests.class); + + Map publisherMap = new HashMap<>(); + + for (int i = 0; i < 10; i++) { + Publisher publisher = new Publisher(); + + String connectId = "123.123.123." + i; + publisher.setAppName("APP"); + //ZONE MUST BE CURRENT SESSION ZONE + publisher.setCell("CELL"); + publisher.setClientId(connectId); + publisher.setDataId(MockSyncDataHandler.dataId); + publisher.setGroup(DEFAULT_GROUP); + publisher.setInstanceId(DEFAULT_INSTANCE_ID); + publisher.setRegisterId(UUID.randomUUID().toString()); + publisher.setProcessId(connectId); + publisher.setVersion(Long.valueOf(i)); + + //registerTimestamp must happen from server,client time maybe different cause pub and unPublisher fail + publisher.setRegisterTimestamp(System.currentTimeMillis()); + + publisher.setClientRegisterTimestamp(System.currentTimeMillis()); + publisher.setSourceAddress(new URL(connectId, 5050)); + + publisher.setClientVersion(ClientVersion.StoreData); + + DataInfo dataInfo = new DataInfo(DEFAULT_INSTANCE_ID, MockSyncDataHandler.dataId, + DEFAULT_GROUP); + publisher.setDataInfoId(dataInfo.getDataInfoId()); + + ServerDataBox serverDataBox = new ServerDataBox(""); + List list = new ArrayList<>(); + list.add(serverDataBox); + + publisher.setDataList(list); + publisherMap.put(publisher.getRegisterId(), publisher); + } + + datum = new Datum(); + datum.setDataId(MockSyncDataHandler.dataId); + datum.setInstanceId(DEFAULT_INSTANCE_ID); + datum.setGroup(DEFAULT_GROUP); + //no datum set version current timestamp + datum.setVersion(DatumVersionUtil.nextId()); + datum.setPubMap(publisherMap); + datum.setDataCenter(ValueConstants.DEFAULT_DATA_CENTER); + + // open sync port and connect it + dataSyncServer = boltExchange.open(new URL(NetUtil.getLocalAddress().getHostAddress(), + TEST_SYNC_PORT), new ChannelHandler[] { new MockGetDataHandler(), + dataApplicationContext.getBean(DataSyncServerConnectionHandler.class) }); + + for (int i = 0; i < connectNum; i++) { + + URL urltemp = new URL(LOCAL_ADDRESS, TEST_SYNC_PORT); + + Client client = boltExchange.connect(Exchange.DATA_SERVER_TYPE + i, urltemp, + new ChannelHandler[] { new MockDataChangeRequestHandler() }); + + BoltChannel boltChannel = (BoltChannel) client.getChannel(urltemp); + + String key = boltChannel.getConnection().getLocalIP() + ":" + + boltChannel.getConnection().getLocalPort(); + boltChannelMap.put(key, boltChannel); + System.out.println("testsyncserver remote connect remote:" + + boltChannel.getConnection().getRemoteAddress() + " local:" + key); + } + + for (int i = 0; i < connectNum; i++) { + + URL urltemp = new URL(LOCAL_ADDRESS, TEST_SYNC_PORT); + + Client client = boltExchange.connect(Exchange.DATA_SERVER_TYPE + i + 10, urltemp, + new ChannelHandler[] { new MockDataChangeRequestHandler() }); + + BoltChannel boltChannel = (BoltChannel) client.getChannel(urltemp); + + String key = boltChannel.getConnection().getLocalIP() + ":" + + boltChannel.getConnection().getLocalPort(); + boltChannelMapInt.put(i, boltChannel); + System.out.println("testsyncserver remote connect remote:" + + boltChannel.getConnection().getRemoteAddress() + " local:" + key); + } + + Thread.sleep(500); + } + + @Ignore + @Test + public void doTest() throws Exception { + + ThreadPoolExecutor executor = new ThreadPoolExecutor(threadNum, threadNum, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new NamedThreadFactory("TestSessionNotify")); + for (int i = 0; i < idNum; i++) { + + int finalI = i; + executor.submit(()->{ + + // post sync data request + DataChangeRequest request = new DataChangeRequest(DataInfo.toDataInfoId( + MockSyncDataHandler.dataId, DEFAULT_INSTANCE_ID, DEFAULT_GROUP), LOCAL_DATACENTER, + finalI); + + boltChannelMap.forEach((connect,boltChannel)->{ + + CommonResponse commonResponse = null; + try { + dataSyncServer.sendCallback(dataSyncServer + .getChannel(new URL(boltChannel.getConnection().getLocalIP(), boltChannel.getConnection().getLocalPort())), request, + new NotifyCallback(boltChannel.getConnection(),request),3000); + //assertTrue(commonResponse.isSuccess()); + } catch (Exception e) { + e.printStackTrace(); + } + }); + }); + } + + while (true){ + TimeUnit.SECONDS.sleep(10); + } + + } + + public static class MockDataChangeRequestHandler extends + AbstractServerHandler { + + @Override + public void checkParam(DataChangeRequest request) throws RuntimeException { + + } + + @Override + public Object doHandle(Channel channel, DataChangeRequest request) { + + DataChangeRequest dataChangeRequest = (DataChangeRequest) request; + + dataChangeRequest.setDataCenter(dataChangeRequest.getDataCenter()); + dataChangeRequest.setDataInfoId(dataChangeRequest.getDataInfoId()); + long version = dataChangeRequest.getVersion(); + + + //update cache when change + sessionCacheService.invalidate(new Key(KeyType.OBJ, DatumKey.class.getName(), new DatumKey( + dataChangeRequest.getDataInfoId(), dataChangeRequest.getDataCenter()))); + + + try { + boolean resultT = sessionInterests.checkInterestVersions( + dataChangeRequest.getDataCenter(), dataChangeRequest.getDataInfoId(), + dataChangeRequest.getVersion()); + //if (!resultT) { + // return null; + //} + + System.out.println(String.format( + "Data version has change,and will fetch to update!Request=%s,URL=%s", + dataChangeRequest, channel.getRemoteAddress())); + //TimeUnit.MILLISECONDS.sleep(2000); + + //fireChangFetch(dataChangeRequest); + //if(version%3==0) { + executor.submit(() -> { + + Object result = boltClientFetch.sendSync(boltChannelMapInt.get(Math.abs((MockSyncDataHandler.dataId+version).hashCode() % connectNum)), + new GetDataRequest(MockSyncDataHandler.dataId, DEFAULT_DATA_CENTER), 1000); + GenericResponse genericResponse = (GenericResponse) result; + if (genericResponse.isSuccess()) { + Map map = (Map) genericResponse.getData(); + if (map == null || map.isEmpty()) { + System.out.println(String.format("GetDataRequest get response contains no datum!dataInfoId=%s", + MockSyncDataHandler.dataId)); + } else { + System.out.println(String.format("GetDataRequest get response contains datum!dataInfoId=%s,size=%s", + MockSyncDataHandler.dataId, map.get(DEFAULT_DATA_CENTER).getPubMap().size())); + } + } else { + throw new RuntimeException( + String.format("GetDataRequest has got fail response!dataInfoId:%s msg:%s", MockSyncDataHandler.dataId, + genericResponse.getMessage())); + } + }); + //} + + } catch (Exception e) { + //LOGGER.error("DataChange Request error!", e); + throw new RuntimeException("DataChangeRequest Request error!", e); + } + //System.out.println(String.format("There are not Subscriber Existed! Who are interest with dataInfoId %s !", + // request.getDataInfoId())); + return null; + } + + @Override + public GenericResponse buildFailedResponse(String msg) { + return new GenericResponse().fillFailed(msg); + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Class interest() { + return DataChangeRequest.class; + } + } + + private static class MockGetDataHandler extends AbstractServerHandler { + public void checkParam(GetDataRequest request) throws RuntimeException { + ParaCheckUtil.checkNotBlank(request.getDataInfoId(), "GetDataRequest.dataInfoId"); + } + + @Override + public Object doHandle(Channel channel, GetDataRequest request) { + Map map = new HashMap<>(); + map.put(DEFAULT_DATA_CENTER, datum); + + return new GenericResponse>().fillSucceed(map); + } + + @Override + public GenericResponse> buildFailedResponse(String msg) { + return new GenericResponse>().fillFailed(msg); + } + + @Override + public HandlerType getType() { + return HandlerType.PROCESSER; + } + + @Override + public Class interest() { + return GetDataRequest.class; + } + + @Override + protected Node.NodeType getConnectNodeType() { + return Node.NodeType.DATA; + } + } + + private class NotifyCallback implements CallbackHandler { + + private int retryTimes = 0; + private Connection connection; + private DataChangeRequest request; + + public NotifyCallback(Connection connection, DataChangeRequest request) { + this.connection = connection; + this.request = request; + } + + @Override + public void onCallback(Channel channel, Object message) { + CommonResponse result = (CommonResponse) message; + if (result != null && !result.isSuccess()) { + System.out + .println(String + .format( + "response not success when notify sessionServer(%s), retryTimes=%s, request=%s, response=%s", + connection.getRemoteAddress(), retryTimes, request, result)); + //onFailed(this); + } + } + + @Override + public void onException(Channel channel, Throwable e) { + System.err.println(String.format( + "exception when notify sessionServer(%s), retryTimes=%s, request=%s,error=%s", + connection.getRemoteAddress(), retryTimes, request, e)); + e.printStackTrace(); + //onFailed(this); + } + + @Override + public Executor getExecutor() { + return null; + } + + } +} diff --git a/test/src/test/java/com/alipay/sofa/registry/test/sync/SyncDataHandlerTest.java b/test/src/test/java/com/alipay/sofa/registry/test/sync/SyncDataHandlerTest.java index 074960dc9..e9985e30b 100644 --- a/test/src/test/java/com/alipay/sofa/registry/test/sync/SyncDataHandlerTest.java +++ b/test/src/test/java/com/alipay/sofa/registry/test/sync/SyncDataHandlerTest.java @@ -51,11 +51,13 @@ public void doTest() throws Exception { String value = "test publish"; PublisherRegistration registration = new PublisherRegistration(dataId); registryClient1.register(registration, value); - Thread.sleep(500L); + Thread.sleep(2000L); // request syncData DataNodeExchanger dataNodeExchanger = dataApplicationContext.getBean("dataNodeExchanger", DataNodeExchanger.class); + URL url = new URL(LOCAL_ADDRESS, syncDataPort); + dataNodeExchanger.connect(url); GenericResponse genericResponse = (GenericResponse) dataNodeExchanger.request( new Request() { @Override @@ -67,7 +69,7 @@ public Object getRequestBody() { @Override public URL getRequestUrl() { - return new URL(LOCAL_ADDRESS, syncDataPort); + return url; } }).getResult(); diff --git a/test/src/test/java/com/alipay/sofa/registry/test/task/DataChangeFetchCloudTaskTest.java b/test/src/test/java/com/alipay/sofa/registry/test/task/DataChangeFetchCloudTaskTest.java new file mode 100644 index 000000000..cb049bcaa --- /dev/null +++ b/test/src/test/java/com/alipay/sofa/registry/test/task/DataChangeFetchCloudTaskTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.registry.test.task; + +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.junit4.SpringRunner; + +import com.alipay.sofa.registry.client.api.model.RegistryType; +import com.alipay.sofa.registry.client.api.registration.PublisherRegistration; +import com.alipay.sofa.registry.client.api.registration.SubscriberRegistration; +import com.alipay.sofa.registry.common.model.constants.ValueConstants; +import com.alipay.sofa.registry.common.model.dataserver.Datum; +import com.alipay.sofa.registry.common.model.store.DataInfo; +import com.alipay.sofa.registry.core.model.ScopeEnum; +import com.alipay.sofa.registry.server.session.bootstrap.SessionServerConfig; +import com.alipay.sofa.registry.server.session.cache.CacheService; +import com.alipay.sofa.registry.server.session.scheduler.ExecutorManager; +import com.alipay.sofa.registry.server.session.scheduler.task.DataChangeFetchCloudTask; +import com.alipay.sofa.registry.server.session.scheduler.task.PushTaskClosure; +import com.alipay.sofa.registry.server.session.scheduler.task.SessionTask; +import com.alipay.sofa.registry.server.session.store.Interests; +import com.alipay.sofa.registry.task.batcher.TaskProcessor.ProcessingResult; +import com.alipay.sofa.registry.task.listener.TaskEvent; +import com.alipay.sofa.registry.task.listener.TaskListenerManager; +import com.alipay.sofa.registry.test.BaseIntegrationTest; + +/** + * @author kezhu.wukz + * @since 2020/03/03 + */ +@RunWith(SpringRunner.class) +public class DataChangeFetchCloudTaskTest extends BaseIntegrationTest { + + private static String testDataInfoId; + private static String testDataId; + + @Before + public void beforeDataChangeFetchCloudTaskTest() throws InterruptedException { + testDataId = "DataChangeFetchCloudTaskTest-" + System.currentTimeMillis(); + testDataInfoId = DataInfo.toDataInfoId(testDataId, ValueConstants.DEFAULT_INSTANCE_ID, + ValueConstants.DEFAULT_GROUP); + + PublisherRegistration registration = new PublisherRegistration(testDataId); + registryClient1.register(registration, value); + Thread.sleep(2000L); + + SubscriberRegistration subReg = new SubscriberRegistration(testDataId, + new MySubscriberDataObserver()); + subReg.setScopeEnum(ScopeEnum.dataCenter); + + registryClient1.register(subReg); + + Thread.sleep(5000L); + } + + @After + public void afterDataChangeFetchCloudTaskTest() { + //remove sub + registryClient1.unregister(testDataId, ValueConstants.DEFAULT_GROUP, + RegistryType.SUBSCRIBER); + registryClient1 + .unregister(testDataId, ValueConstants.DEFAULT_GROUP, RegistryType.PUBLISHER); + } + + @Test + public void getTaskClosureTest() { + String dataCenter = "MockDC"; + + Interests sessionInterests = (Interests) sessionApplicationContext + .getBean("sessionInterests"); + SessionServerConfig sessionServerConfig = (SessionServerConfig) sessionApplicationContext + .getBean("sessionServerConfig"); + TaskListenerManager taskListenerManager = (TaskListenerManager) sessionApplicationContext + .getBean("taskListenerManager"); + ExecutorManager executorManager = (ExecutorManager) sessionApplicationContext + .getBean("executorManager"); + CacheService sessionCacheService = (CacheService) sessionApplicationContext + .getBean("sessionCacheService"); + TaskEvent event = new TaskEvent(testDataInfoId, + TaskEvent.TaskType.DATA_CHANGE_FETCH_CLOUD_TASK); + + SessionTask dataChangeFetchTask = new DataChangeFetchCloudTask(sessionServerConfig, + taskListenerManager, sessionInterests, executorManager, sessionCacheService); + dataChangeFetchTask.setTaskEvent(event); + + //put a new dataCenter, version set 1 + + sessionInterests.checkAndUpdateInterestVersions(dataCenter, testDataInfoId, 1L); + List dataCenters = sessionInterests.getDataCenters(); + assertTrue(dataCenters.size() > 0); + assertTrue(dataCenters.contains(dataCenter)); + + // version will uodate to 0 + Map datumMap = new HashMap<>(); + { + Datum datum = new Datum(); + datum.setVersion(2); + datumMap.put("testDataCenter1", datum); + } + { + Datum datum = new Datum(); + datum.setVersion(2); + datumMap.put("testDataCenter2", datum); + } + PushTaskClosure taskClosure = ((DataChangeFetchCloudTask) dataChangeFetchTask) + .getTaskClosure(datumMap); + taskClosure.run(ProcessingResult.Success, dataChangeFetchTask); + + // check version should be 0 + assertTrue(sessionInterests.checkInterestVersions(dataCenter, testDataInfoId, 1L)); + + } +}