Skip to content

Commit 354163a

Browse files
committed
perftest: enable CQ events support for WRITE_IMM operations
Remove the restriction that prevented WRITE_IMM operations from using completion queue events. As WRITE_IMM posts to the reciever CQ, there's no reason not to support the event parameter (-e) as in WRITE. Changes include: - Remove WRITE_IMM exclusion from events parameter parsing - Add CQ notification setup for write_bw and write_lat tools - Implement event notification in run_iter_lat_write_imm() - Update polling logic to respect use_event flag Reviewed-by: Michael Margolin <mrgolin@amazon.com> Reviewed-by: Yonatan Nachum <ynachum@amazon.com> Signed-off-by: Ramon Fried <friedram@amazon.com>
1 parent 4bee61f commit 354163a

File tree

4 files changed

+47
-9
lines changed

4 files changed

+47
-9
lines changed

src/perftest_parameters.c

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
321321
printf(" -D, --duration ");
322322
printf(" Run test for a customized period of seconds. (SYMMETRIC)\n");
323323

324-
if (verb != WRITE && verb != WRITE_IMM && connection_type != RawEth) {
324+
if (verb != WRITE && connection_type != RawEth) {
325325
printf(" -e, --events ");
326326
printf(" Sleep on CQ events (default poll)\n");
327327

@@ -1790,6 +1790,12 @@ static void force_dependecies(struct perftest_parameters *user_param)
17901790
}
17911791
}
17921792

1793+
if (user_param->use_event == ON && user_param->verb == WRITE) {
1794+
printf(RESULT_LINE);
1795+
fprintf(stderr," Events feature not available on WRITE verb\n");
1796+
exit(1);
1797+
}
1798+
17931799
if ((user_param->use_srq && (user_param->tst == LAT || user_param->machine == SERVER || user_param->duplex == ON)) || user_param->use_xrc)
17941800
user_param->srq_exists = 1;
17951801

@@ -2949,12 +2955,8 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
29492955
return FAILURE;
29502956
}
29512957
break;
2952-
case 'e': user_param->use_event = ON;
2953-
if (user_param->verb == WRITE || user_param->verb == WRITE_IMM) {
2954-
fprintf(stderr," Events feature not available on WRITE verb\n");
2955-
free(duplicates_checker);
2956-
return FAILURE;
2957-
}
2958+
case 'e':
2959+
user_param->use_event = ON;
29582960
break;
29592961
case 'X':
29602962
if (user_param->verb == WRITE || user_param->verb == WRITE_IMM) {

src/perftest_resources.c

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5064,7 +5064,7 @@ int run_iter_lat_write_imm(struct pingpong_context *ctx,struct perftest_paramete
50645064
uint64_t scnt = 0;
50655065
uint64_t ccnt = 0;
50665066
uint64_t rcnt = 0;
5067-
int ne;
5067+
int ne = 0;
50685068
interr = 0;
50695069
volatile char *post_buf = NULL;
50705070

@@ -5111,8 +5111,15 @@ int run_iter_lat_write_imm(struct pingpong_context *ctx,struct perftest_paramete
51115111
if ((rcnt < user_param->iters || user_param->test_type == DURATION) && !(scnt < 1 && user_param->machine == SERVER)) {
51125112
rcnt++;
51135113

5114+
if (user_param->use_event) {
5115+
if (ctx_notify_events(ctx->recv_channel)) {
5116+
fprintf(stderr , " Failed to notify events to CQ\n");
5117+
return 1;
5118+
}
5119+
}
5120+
51145121
/* Poll for a completion */
5115-
do { ne = ibv_poll_cq(ctx->recv_cq, 1, &wc); } while (ne == 0 && !(user_param->test_type == DURATION && user_param->state == END_STATE));
5122+
do { ne = ibv_poll_cq(ctx->recv_cq, 1, &wc); } while (!user_param->use_event && ne == 0 && !(user_param->test_type == DURATION && user_param->state == END_STATE));
51165123
if (ne > 0) {
51175124
if (wc.status != IBV_WC_SUCCESS) {
51185125
//coverity[uninit_use_in_call]
@@ -5172,6 +5179,13 @@ int run_iter_lat_write_imm(struct pingpong_context *ctx,struct perftest_paramete
51725179

51735180
if (ccnt < user_param->iters || user_param->test_type == DURATION) {
51745181

5182+
if (user_param->use_event && ne == 0) {
5183+
if (ctx_notify_events(ctx->send_channel)) {
5184+
fprintf(stderr, "Couldn't request CQ notification\n");
5185+
return 1;
5186+
}
5187+
}
5188+
51755189
do { ne = ibv_poll_cq(ctx->send_cq, 1, &wc); } while (ne == 0);
51765190

51775191
if(ne > 0) {

src/write_bw.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,17 @@ int main(int argc, char *argv[])
222222
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
223223
}
224224

225+
if (user_param.use_event) {
226+
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
227+
fprintf(stderr, " Couldn't request CQ notification\n");
228+
goto destroy_context;
229+
}
230+
if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
231+
fprintf(stderr, " Couldn't request CQ notification\n");
232+
goto destroy_context;
233+
}
234+
}
235+
225236
/* An additional handshake is required after moving qp to RTR. */
226237
if (ctx_hand_shake(&user_comm,&my_dest[0],&rem_dest[0])) {
227238
fprintf(stderr," Failed to exchange data between server and clients\n");

src/write_lat.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,17 @@ int main(int argc, char *argv[])
243243
ctx_print_pingpong_data(&rem_dest[i],&user_comm);
244244
}
245245

246+
if (user_param.use_event) {
247+
if (ibv_req_notify_cq(ctx.send_cq, 0)) {
248+
fprintf(stderr, " Couldn't request CQ notification\n");
249+
goto destroy_context;
250+
}
251+
if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
252+
fprintf(stderr, " Couldn't request CQ notification\n");
253+
goto destroy_context;
254+
}
255+
}
256+
246257
/* An additional handshake is required after moving qp to RTR. */
247258
if (ctx_hand_shake(&user_comm,my_dest,rem_dest)) {
248259
fprintf(stderr,"Failed to exchange data between server and clients\n");

0 commit comments

Comments
 (0)