From db8075001ff494366c46be164ed963a2acbf56aa Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 9 Jan 2025 13:30:59 +0800 Subject: [PATCH 1/3] pom file adds a new property data_region_per_data_node --- .github/workflows/daily-it.yml | 56 +++++++++++++++++++ integration-test/pom.xml | 2 + .../iotdb/it/env/cluster/ClusterConstant.java | 2 + .../it/env/cluster/node/DataNodeWrapper.java | 6 ++ 4 files changed, 66 insertions(+) diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml index 3783e563e35f..0f8bf9fbe3d9 100644 --- a/.github/workflows/daily-it.yml +++ b/.github/workflows/daily-it.yml @@ -49,3 +49,59 @@ jobs: name: cluster-log-java${{ matrix.java }}-${{ runner.os }} path: integration-test/target/cluster-logs retention-days: 3 + SingleRegionTableModel: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 8, 17 ] + runs-on: [ self-hosted, iotdb ] + # group: self-hosted + # labels: iotdb + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: 17 + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Adjust network dynamic TCP ports range + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + netsh int ipv4 set dynamicport tcp start=32768 num=32768 + netsh int ipv4 set dynamicport udp start=32768 num=32768 + netsh int ipv6 set dynamicport tcp start=32768 num=32768 + netsh int ipv6 set dynamicport udp start=32768 num=32768 + - name: Adjust Linux kernel somaxconn + if: ${{ runner.os == 'Linux' }} + shell: bash + run: sudo sysctl -w net.core.somaxconn=65535 + # - name: Adjust Mac kernel somaxconn + # if: ${{ runner.os == 'macOS' }} + # shell: bash + # run: sudo sysctl -w kern.ipc.somaxconn=65535 + - name: IT/UT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=2 -DDataNodeMaxHeapSize=1024 -DintegrationTest.dataRegionPerDataNode=1\ + -pl integration-test \ + -am -PTableSimpleIT + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }} + path: integration-test/target/cluster-logs + retention-days: 3 \ No newline at end of file diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 02c54ceb8a9f..43e30ace4ea9 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -37,6 +37,7 @@ 200m true true + 0 Simple @@ -300,6 +301,7 @@ ${integrationTest.testEnv} ${integrationTest.randomSelectWriteNode} ${integrationTest.readAndVerifyWithMultiNode} + ${integrationTest.dataRegionPerDataNode} ${lightWeightStandaloneMode.configNodeNumber} ${lightWeightStandaloneMode.dataNodeNumber} ${lightWeightStandaloneMode.configNodeConsensus} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java index 8b9a3816bcf1..2516bf665614 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java @@ -43,6 +43,7 @@ public class ClusterConstant { public static final String DEFAULT_DATA_NODE_PROPERTIES = "DefaultDataNodeProperties"; public static final String DEFAULT_DATA_NODE_COMMON_PROPERTIES = "DefaultDataNodeCommonProperties"; + public static final String DATA_REGION_PER_DATANODE = "integrationTest.dataRegionPerDataNode"; // Cluster Configurations public static final String CLUSTER_CONFIGURATIONS = "ClusterConfigurations"; @@ -202,6 +203,7 @@ public class ClusterConstant { "schema_region_ratis_log_appender_buffer_size_max"; public static final String DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX = "data_region_ratis_log_appender_buffer_size_max"; + public static final String DATA_REGION_PER_DATA_NODE = "data_region_per_data_node"; // Paths public static final String USER_DIR = "user.dir"; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 127f9b331857..aca112048cdd 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -36,6 +36,8 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATANODE_MAX_HEAP_SIZE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_NODE_NAME; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATANODE; +import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_PER_DATA_NODE; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR; import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_DATA_NODE_COMMON_PROPERTIES; @@ -82,6 +84,8 @@ public class DataNodeWrapper extends AbstractNodeWrapper { private final String defaultCommonPropertiesFile; + private final int regionPerDataNode; + public DataNodeWrapper( final String seedConfigNode, final String testClassName, @@ -103,6 +107,7 @@ public DataNodeWrapper( EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_PROPERTIES, clusterIndex); this.defaultCommonPropertiesFile = EnvUtils.getFilePathFromSysVar(DEFAULT_DATA_NODE_COMMON_PROPERTIES, clusterIndex); + this.regionPerDataNode = EnvUtils.getIntFromSysVar(DATA_REGION_PER_DATANODE, 0, clusterIndex); // Initialize mutable properties reloadMutableFields(); @@ -209,6 +214,7 @@ protected void reloadMutableFields() { mutableNodeProperties.setProperty(WAL_BUFFER_SIZE_IN_BYTE, "16777216"); mutableNodeProperties.setProperty(SCHEMA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); mutableNodeProperties.setProperty(DATA_REGION_RATIS_LOG_APPENDER_BUFFER_SIZE_MAX, "8388608"); + mutableNodeProperties.setProperty(DATA_REGION_PER_DATA_NODE, String.valueOf(regionPerDataNode)); } @Override From f5390a18a4f384a8c56d569a22d12c71a4af152c Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Tue, 21 Jan 2025 12:25:38 +0800 Subject: [PATCH 2/3] Daily_IT removes dependency --- .github/workflows/daily-it.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml index 0f8bf9fbe3d9..7add0904050c 100644 --- a/.github/workflows/daily-it.yml +++ b/.github/workflows/daily-it.yml @@ -31,10 +31,20 @@ jobs: with: distribution: liberica java-version: ${{ matrix.java }} + - name: Adjust network dynamic TCP ports range + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + netsh int ipv4 set dynamicport tcp start=32768 num=32768 + netsh int ipv4 set dynamicport udp start=32768 num=32768 + netsh int ipv6 set dynamicport tcp start=32768 num=32768 + netsh int ipv6 set dynamicport udp start=32768 num=32768 + - name: Adjust Linux kernel somaxconn + if: ${{ runner.os == 'Linux' }} + shell: bash + run: sudo sysctl -w net.core.somaxconn=65535 - name: IT/UT Test shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml run: | mvn clean verify \ -P with-integration-tests \ @@ -64,13 +74,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: liberica - java-version: 17 - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- + java-version: ${{ matrix.java }} - name: Adjust network dynamic TCP ports range if: ${{ runner.os == 'Windows' }} shell: pwsh From 3e149e60a724607d27a1805655c88b1c6ccd5959 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 23 Jan 2025 16:32:56 +0800 Subject: [PATCH 3/3] Delete the case about runner.os == MacOS --- .github/workflows/daily-it.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml index 7add0904050c..6c54234f05cd 100644 --- a/.github/workflows/daily-it.yml +++ b/.github/workflows/daily-it.yml @@ -87,10 +87,6 @@ jobs: if: ${{ runner.os == 'Linux' }} shell: bash run: sudo sysctl -w net.core.somaxconn=65535 - # - name: Adjust Mac kernel somaxconn - # if: ${{ runner.os == 'macOS' }} - # shell: bash - # run: sudo sysctl -w kern.ipc.somaxconn=65535 - name: IT/UT Test shell: bash # we do not compile client-cpp for saving time, it is tested in client.yml