feat: Optimize the load balancing logic to achieve rebalancing after node failure recovery. (#291)

* fix: load properties issue in windows

* feat: supports rebalance when node down

* fix: WebSocket statement timeout setting resets transport's timeout

Closes TD-38565

* --amend

* --amend

* chore: fix unit test issues

* chore: add unit tests

Author: sheyanjie-qq

.github/workflows/build.yml

@@ -250,6 +250,5 @@ jobs:
         with:
           name: service-logs
           path: |
-            /var/log/taosd.log
-            /var/log/taosadapter.log
-          retention-days: 1
+            /var/log/taos/**/*
+          retention-days: 7

.github/workflows/compatibility-3360.yml

@@ -1,6 +1,8 @@
 name: build-3360
+
 permissions:
   contents: read
+
 on:
   push:
     branches:
@@ -69,6 +71,13 @@ jobs:
           TDENGINE_CLOUD_URL: ${{ secrets.TDENGINE_CLOUD_URL }}
           TD_3360_TEST: 'true'
         run: mvn -B clean verify --file pom.xml
-
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: service-logs
+          path: |
+            /var/log/taos/**/*
+          retention-days: 7
 
 

.github/workflows/sonar.yml

@@ -1,6 +1,7 @@
 name: sonar
 permissions:
   contents: read
+
 on:
   push:
     branches:

deploy-pom.xml

@@ -49,6 +49,7 @@
         <junit.version>4.13.2</junit.version>
         <mockito.version>4.11.0</mockito.version>
         <slf4j.version>1.7.36</slf4j.version>
+        <logback.version>1.3.16</logback.version>
     </properties>
 
     <dependencies>
@@ -64,6 +65,18 @@
             <version>${mockito.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-inline</artifactId>
+            <version>${mockito.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback.version}</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

pom.xml

@@ -44,8 +44,9 @@
         <netty-all.version>4.1.127.Final</netty-all.version>
         <junit.version>4.13.2</junit.version>
         <mockito.version>4.11.0</mockito.version>
-        <slf4j.version>1.7.36</slf4j.version>
-
+        <byte-buddy.version>1.12.22</byte-buddy.version>
+        <slf4j.version>2.0.17</slf4j.version>
+        <logback.version>1.3.16</logback.version>
     </properties>
     <dependencies>
         <dependency>
@@ -91,6 +92,18 @@
             <version>${mockito.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>net.bytebuddy</groupId>
+            <artifactId>byte-buddy</artifactId>
+            <version>${byte-buddy.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback.version}</version>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>
 
@@ -198,11 +211,19 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-                <version>3.0.0-M6</version>
+                <version>3.2.5</version>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
                 <executions>
                     <execution>
                         <id>parallel-tests</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
                         <configuration>
+                            <skip>false</skip>
                             <forkCount>10</forkCount>
                             <reuseForks>false</reuseForks>
 
@@ -240,6 +261,7 @@
                             <goal>test</goal>
                         </goals>
                         <configuration>
+                            <skip>false</skip>
                             <forkCount>1</forkCount>
                             <reuseForks>false</reuseForks>
                             <parallel>none</parallel>

src/main/java/com/taosdata/jdbc/AbstractConnection.java

@@ -42,6 +42,10 @@ public void registerStatement(Long stmtId, Statement stmt) {
         this.statementsMap.put(stmtId, stmt);
     }
 
+    public boolean canRebalanced() {
+        return this.statementsMap.isEmpty();
+    }
+
     @Override
     public abstract Statement createStatement() throws SQLException;
 

src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java

@@ -1714,7 +1714,14 @@ private ColumnMetaData buildSuperTableNameMeta(int colIndex) {
     }
 
     private String generateDescribeSql(String dbName, String tableName) throws SQLException {
-        return "describe " + dbName + "." + getIdentifierQuoteString() + tableName + getIdentifierQuoteString();
+        return "describe " +
+                getIdentifierQuoteString() +
+                dbName +
+                getIdentifierQuoteString() +
+                "." +
+                getIdentifierQuoteString() +
+                tableName +
+                getIdentifierQuoteString();
     }
 
 

src/main/java/com/taosdata/jdbc/AbstractDriver.java

@@ -62,7 +62,7 @@ protected Connection getWSConnection(String url, ConnectionParam param, Properti
                 log.error("Error serializing ConnectionParam", e);
             }
         }
-        InFlightRequest inFlightRequest = new InFlightRequest(param.getRequestTimeout(), param.getMaxRequest());
+        InFlightRequest inFlightRequest = new InFlightRequest(param.getMaxRequest());
         param.setTextMessageHandler(message -> {
             try {
                 log.trace("received message: {}", message);
@@ -88,8 +88,6 @@ protected Connection getWSConnection(String url, ConnectionParam param, Properti
             FetchBlockData fetchBlockData = FetchDataUtil.getFetchMap().get(id);
             if (null != fetchBlockData) {
                 Utils.retainByteBuf(byteBuf);
-                byte[] bytes = new byte[byteBuf.readableBytes()];
-                byteBuf.getBytes(byteBuf.readerIndex(), bytes);
 
                 FetchBlockNewResp fetchBlockResp = new FetchBlockNewResp(byteBuf);
                 try {

src/main/java/com/taosdata/jdbc/SchemalessWriter.java

@@ -156,7 +156,7 @@ private void init(String host, String port, String user, String password, String
                     .setRequestTimeout(timeout)
                     .setUseSsl(useSSL)
                     .build();
-            InFlightRequest inFlightRequest = new InFlightRequest(timeout, 20);
+            InFlightRequest inFlightRequest = new InFlightRequest(20);
             param.setTextMessageHandler(message -> {
                 try {
                     JsonNode jsonObject = JsonUtil.getObjectReader().readTree(message);

src/main/java/com/taosdata/jdbc/TSDBConstants.java

@@ -93,7 +93,6 @@ public abstract class TSDBConstants {
     public static final int DEFAULT_MESSAGE_WAIT_TIMEOUT = 60_000;
     public static final boolean DEFAULT_BATCH_ERROR_IGNORE = false;
 
-
     public static final short MAX_UNSIGNED_BYTE = 255;
     public static final int MAX_UNSIGNED_SHORT = 65535;
     public static final long MAX_UNSIGNED_INT = 4294967295L;