Skip to content

Commit 67edba7

Browse files
sheyanjie-qqsheyanjie-qq
authored
perf: improve isValid performace (#281)
* chore: improve tests * feat: use cache to improve isValid perf
1 parent 87f27e8 commit 67edba7

19 files changed

+861
-72
lines changed

‎.github/workflows/build.yml‎

Lines changed: 167 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,56 +18,199 @@ on:
1818
- '*.md'
1919
- '*.txt'
2020

21+
env:
22+
CACHE_VERSION: v1 # Update this version when cache strategy needs refresh
23+
2124
jobs:
2225
build:
2326
strategy:
2427
matrix:
25-
# os: [ ubuntu-latest,macos-latest,windows-latest ]
2628
os: [ Ubuntu-22.04 ]
2729
java: [ 8 ]
2830
maven: [ '3.6.3' ]
2931
runs-on: ${{ matrix.os }}
3032
steps:
31-
- name: checkout TDengine
33+
- name: Checkout TDengine
3234
uses: actions/checkout@v3
3335
with:
3436
repository: 'taosdata/TDengine'
3537
path: 'TDengine'
36-
ref: ${{ github.base_ref }}
38+
ref: ${{ github.event.pull_request.base.ref || github.ref_name }}
39+
40+
- name: Get TDengine branch and commit info
41+
id: tdengine-info
42+
run: |
43+
cd TDengine
44+
echo "commit_id=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
45+
echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
46+
echo "short_sha=$(git rev-parse --short=8 HEAD)" >> $GITHUB_OUTPUT
47+
48+
- name: Restore TDengine debug directory cache
49+
uses: actions/cache/restore@v4
50+
id: restore-cache
51+
with:
52+
path: TDengine/debug
53+
key: ${{ runner.os }}-tdengine-debug-${{ env.CACHE_VERSION }}-${{ steps.tdengine-info.outputs.branch }}-${{ steps.tdengine-info.outputs.commit_id }}
54+
restore-keys: |
55+
${{ runner.os }}-tdengine-debug-${{ env.CACHE_VERSION }}-${{ steps.tdengine-info.outputs.branch }}-
56+
${{ runner.os }}-tdengine-debug-${{ env.CACHE_VERSION }}-
57+
58+
- name: Install build dependencies
59+
if: steps.cache-debug-dir.outputs.cache-hit != 'true'
60+
run: |
61+
sudo apt-get update
62+
sudo apt-get install -y libgeos-dev build-essential cmake gcc g++ make
63+
64+
- name: Build TDengine (if cache missed)
65+
if: steps.cache-debug-dir.outputs.cache-hit != 'true'
66+
id: build-step
67+
run: |
68+
cd TDengine
69+
mkdir -p debug
70+
cd debug
71+
cmake .. -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=false
72+
make -j $(nproc)
73+
74+
- name: Save cache on build success
75+
if: always() && steps.build-step.conclusion == 'success'
76+
uses: actions/cache/save@v4
77+
with:
78+
path: TDengine/debug
79+
key: ${{ runner.os }}-tdengine-debug-${{ env.CACHE_VERSION }}-${{ steps.tdengine-info.outputs.branch }}-${{ steps.tdengine-info.outputs.commit_id }}
3780

38-
- name: prepare install
39-
run: |
40-
sudo apt-get install -y libgeos-dev
41-
geos-config --version
42-
43-
- name: install TDengine
44-
run: cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=false && make && sudo make install
81+
82+
- name: Verify debug directory
83+
run: |
84+
echo "=== Verifying debug directory ==="
85+
if [ ! -d "TDengine/debug" ]; then
86+
echo "❌ Error: Debug directory not found!"
87+
exit 1
88+
fi
89+
90+
echo "Debug directory contents:"
91+
ls -la TDengine/debug/
92+
93+
if [ ! -d "TDengine/debug/build" ]; then
94+
echo "❌ Error: Build directory not found in debug directory!"
95+
exit 1
96+
fi
97+
98+
file_count=$(find TDengine/debug/build -type f 2>/dev/null | wc -l)
99+
echo "✅ Build directory exists with $file_count files"
100+
101+
- name: Install TDengine from debug directory
102+
run: |
103+
echo "=== Installing TDengine ==="
104+
if [ ! -d "TDengine/debug" ]; then
105+
echo "❌ Error: Debug directory not found for installation!"
106+
exit 1
107+
fi
108+
109+
cd TDengine/debug
110+
echo "Running make install..."
111+
if ! sudo make install; then
112+
echo "❌ Error: make install failed!"
113+
exit 1
114+
fi
115+
echo "✅ TDengine installed successfully"
116+
117+
- name: Verify TDengine installation
118+
run: |
119+
echo "=== Verifying TDengine installation ==="
120+
121+
# Check taosd
122+
if ! which taosd >/dev/null 2>&1; then
123+
echo "❌ Error: taosd not found in PATH!"
124+
exit 1
125+
fi
126+
echo "✅ taosd found: $(which taosd)"
127+
taosd --version || echo "⚠️ Could not get taosd version"
128+
129+
# Check taos
130+
if ! which taos >/dev/null 2>&1; then
131+
echo "❌ Error: taos not found in PATH!"
132+
exit 1
133+
fi
134+
echo "✅ taos found: $(which taos)"
135+
taos --version || echo "⚠️ Could not get taos version"
136+
137+
# Check installation directories
138+
if [ -d "/usr/local/taos/bin" ]; then
139+
echo "✅ TDengine binaries in /usr/local/taos/bin:"
140+
ls -la /usr/local/taos/bin/
141+
else
142+
echo "⚠️ /usr/local/taos/bin/ not found"
143+
fi
144+
145+
echo "✅ TDengine installation verified"
45146
46147
- name: shell
47148
run: |
48149
cat >start.sh<<EOF
49150
ulimit -n 65535 && TAOS_SUPPORT_VNODES=256 taosd
50151
EOF
51-
52152
- name: taosd
53153
run: nohup sudo sh ./start.sh &
54154

55155
- name: start taosadapter
56156
run: sudo taosadapter &
57157

58-
- name: checkout
158+
- name: Check service status
159+
run: |
160+
echo "=== Checking service status ==="
161+
162+
# Check taos processes
163+
if ! pgrep -f "[t]aosd" >/dev/null 2>&1; then
164+
echo "❌ Error: No taosd process found!"
165+
echo "taosd log:"
166+
sudo tail -20 /var/log/taosd.log 2>/dev/null || echo "No taosd log found"
167+
exit 1
168+
fi
169+
echo "✅ taosd process is running"
170+
171+
if ! pgrep -f "[t]aosadapter" >/dev/null 2>&1; then
172+
echo "❌ Error: No taosadapter process found!"
173+
echo "taosadapter log:"
174+
sudo tail -20 /var/log/taosadapter.log 2>/dev/null || echo "No taosadapter log found"
175+
exit 1
176+
fi
177+
echo "✅ taosadapter process is running"
178+
179+
# Check port 6030
180+
if ! sudo netstat -tlnp | grep -q 6030; then
181+
echo "❌ Error: Port 6030 not listening!"
182+
echo "Current listening ports:"
183+
sudo netstat -tlnp | grep LISTEN || echo "No listening ports found"
184+
exit 1
185+
fi
186+
echo "✅ Port 6030 is listening"
187+
188+
# Additional connectivity test
189+
echo "Testing TDengine connectivity..."
190+
if ! timeout 10s taos -s "show dnodes" >/dev/null 2>&1; then
191+
echo "❌ Error: Cannot connect to TDengine!"
192+
echo "Service status:"
193+
ps aux | grep -E "[t]aos"
194+
echo "Port check:"
195+
sudo netstat -tlnp | grep -E "6030|6060" || echo "No TDengine ports found"
196+
exit 1
197+
fi
198+
199+
echo "✅ All services are running correctly"
200+
- name: Checkout JDBC connector
59201
uses: actions/checkout@v4
60202
with:
61203
path: 'jdbc-workspace'
62204

63-
- name: set up java
205+
- name: Set up Java
64206
uses: actions/setup-java@v3
65207
with:
66-
distribution: 'temurin' # See 'Supported distributions' for available options
208+
distribution: 'temurin'
67209
java-version: ${{ matrix.java }}
68210
java-package: jdk
211+
cache: 'maven'
69212

70-
- name: Test
213+
- name: Run JDBC tests
71214
working-directory: jdbc-workspace
72215
env:
73216
TDENGINE_CLOUD_URL: ${{ secrets.TDENGINE_CLOUD_URL }}
@@ -84,5 +227,12 @@ jobs:
84227
verbose: true
85228
root_dir: jdbc-workspace
86229

87-
88-
230+
- name: Upload logs on failure
231+
if: failure()
232+
uses: actions/upload-artifact@v4
233+
with:
234+
name: service-logs
235+
path: |
236+
/var/log/taosd.log
237+
/var/log/taosadapter.log
238+
retention-days: 1

‎deploy-pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
<jackson.version>2.18.4</jackson.version>
4646
<httpclient.version>4.5.14</httpclient.version>
4747
<guava.version>33.5.0-jre</guava.version>
48-
<netty-all.version>4.2.6.Final</netty-all.version>
48+
<netty-all.version>4.1.127.Final</netty-all.version>
4949
<junit.version>4.13.2</junit.version>
5050
<mockito.version>4.11.0</mockito.version>
5151
<slf4j.version>1.7.36</slf4j.version>

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
<jackson.version>2.20.0</jackson.version>
4141
<httpclient.version>4.5.14</httpclient.version>
4242
<guava.version>33.5.0-jre</guava.version>
43-
<netty-all.version>4.2.6.Final</netty-all.version>
43+
<netty-all.version>4.1.127.Final</netty-all.version>
4444
<junit.version>4.13.2</junit.version>
4545
<mockito.version>4.11.0</mockito.version>
4646
<slf4j.version>1.7.36</slf4j.version>

‎src/main/java/com/taosdata/jdbc/TSDBDriver.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ public class TSDBDriver extends AbstractDriver {
109109
*/
110110
public static final String PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT = "messageWaitTimeout";
111111

112+
public static final String PROPERTY_KEY_WS_KEEP_ALIVE_SECONDS = "wsKeepAlive";
112113
/**
113114
* connect mode
114115
*/
@@ -133,7 +134,9 @@ public class TSDBDriver extends AbstractDriver {
133134
public static final String PROPERTY_KEY_APP_IP = "app_ip";
134135
public static final String PROPERTY_KEY_APP_NAME = "app_name";
135136

136-
// for efficient writing
137+
/**
138+
* for efficient writing
139+
*/
137140
public static final String PROPERTY_KEY_COPY_DATA = "copyData";
138141
public static final String PROPERTY_KEY_BATCH_SIZE_BY_ROW = "batchSizeByRow";
139142
public static final String PROPERTY_KEY_CACHE_SIZE_BY_ROW = "cacheSizeByRow";
@@ -142,7 +145,9 @@ public class TSDBDriver extends AbstractDriver {
142145
public static final String PROPERTY_KEY_RETRY_TIMES = "retryTimes";
143146
public static final String PROPERTY_KEY_ASYNC_WRITE = "asyncWrite";
144147

145-
// for stmt bind mode
148+
/**
149+
* for stmt bind mode
150+
*/
146151
public static final String PROPERTY_KEY_PBS_MODE = "pbsMode";
147152

148153
/**

‎src/main/java/com/taosdata/jdbc/rs/ConnectionParam.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ public class ConnectionParam {
5353
private int retryTimes;
5454
private String asyncWrite;
5555
private String pbsMode;
56+
private int wsKeepAlive;
5657

5758
private Consumer<String> textMessageHandler;
5859
private Consumer<ByteBuf> binaryMessageHandler;
@@ -90,6 +91,7 @@ private ConnectionParam(Builder builder) {
9091
this.textMessageHandler = builder.textMessageHandler;
9192
this.binaryMessageHandler = builder.binaryMessageHandler;
9293
this.pbsMode = builder.pbsMode;
94+
this.wsKeepAlive = builder.wsKeepAlive;
9395
}
9496

9597
public void setEndpoints(List<Endpoint> endpoints) {
@@ -206,6 +208,10 @@ public void setAsyncWrite(String asyncWrite) {
206208
public void setPbsMode(String pbsMode) {
207209
this.pbsMode = pbsMode;
208210
}
211+
public void setWsKeepAlive(int wsKeepAlive) {
212+
this.wsKeepAlive = wsKeepAlive;
213+
}
214+
209215
public List<Endpoint> getEndpoints() {
210216
return endpoints;
211217
}
@@ -306,7 +312,9 @@ public String getAsyncWrite() {
306312
public String getPbsMode() {
307313
return pbsMode;
308314
}
309-
315+
public int getWsKeepAlive() {
316+
return wsKeepAlive;
317+
}
310318
public Consumer<String> getTextMessageHandler() {
311319
return textMessageHandler;
312320
}
@@ -482,6 +490,11 @@ public static ConnectionParam getParam(Properties properties) throws SQLExceptio
482490
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "PROPERTY_KEY_PBS_MODE only support line");
483491
}
484492

493+
int wsKeepAlive = Integer.parseInt(properties.getProperty(TSDBDriver.PROPERTY_KEY_WS_KEEP_ALIVE_SECONDS, "300"));
494+
if (wsKeepAlive <= 0){
495+
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid para PROPERTY_KEY_WS_KEEP_ALIVE_SECONDS");
496+
}
497+
485498
return new Builder(endpoints)
486499
.setDatabase(database)
487500
.setCloudToken(cloudToken)
@@ -510,6 +523,7 @@ public static ConnectionParam getParam(Properties properties) throws SQLExceptio
510523
.setRetryTimes(retryTimes)
511524
.setAsyncWrite(asyncWrite)
512525
.setPbsMode(pbsMode)
526+
.setWsKeepAlive(wsKeepAlive)
513527
.build();
514528
}
515529

@@ -545,6 +559,7 @@ public static class Builder {
545559
private int retryTimes;
546560
private String asyncWrite;
547561
private String pbsMode;
562+
private int wsKeepAlive;
548563

549564
private Consumer<String> textMessageHandler;
550565
private Consumer<ByteBuf> binaryMessageHandler;
@@ -684,6 +699,12 @@ public Builder setPbsMode(String pbsMode) {
684699
this.pbsMode = pbsMode;
685700
return this;
686701
}
702+
703+
public Builder setWsKeepAlive(int wsKeepAlive) {
704+
this.wsKeepAlive = wsKeepAlive;
705+
return this;
706+
}
707+
687708
public Builder setTextMessageHandler(Consumer<String> textMessageHandler) {
688709
this.textMessageHandler = textMessageHandler;
689710
return this;

‎src/main/java/com/taosdata/jdbc/utils/StmtUtils.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import com.taosdata.jdbc.TSDBError;
44
import com.taosdata.jdbc.TSDBErrorNumbers;
5-
import com.taosdata.jdbc.rs.ConnectionParam;
65
import com.taosdata.jdbc.ws.Transport;
76
import com.taosdata.jdbc.ws.entity.Code;
87
import com.taosdata.jdbc.ws.entity.Request;
@@ -17,9 +16,8 @@
1716
public class StmtUtils {
1817
private static final Logger log = LoggerFactory.getLogger(StmtUtils.class);
1918
private StmtUtils() {}
20-
public static Stmt2PrepareResp initStmtWithRetry(Transport transport, String sql, ConnectionParam param) throws SQLException {
19+
public static Stmt2PrepareResp initStmtWithRetry(Transport transport, String sql, int retryTimes) throws SQLException {
2120
SQLException lastError = TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN);
22-
int retryTimes = param.isEnableAutoConnect() ? param.getRetryTimes() : 1;
2321
for (int i = 0; i < retryTimes; i++) {
2422
long reqId = ReqId.getReqID();
2523
long stmtId = 0;

0 commit comments

Comments
 (0)