Skip to content

COLL/UCC: Fix aborting with SIGSEGV when persistent request is failed #13440

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 17, 2025
Merged

COLL/UCC: Fix aborting with SIGSEGV when persistent request is failed #13440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .mailmap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,3 +137,5 @@ George Katevenis <gkatev@ics.forth.gr>
Brian Barrett <brian@bbarrett.org>

Andrii Bilokur <abilokur@nvidia.com> B-a-S <abilokur@nvidia.com>

Kento Hasegawa <hasegawa.kento@fujitsu.com> hasegawa.kento <hasegawa.kento@fujitsu.com>
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_allgather.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ int mca_coll_ucc_iallgather(const void *sbuf, size_t scount, struct ompi_datatyp
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc iallgather");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_allgather_init_common(sbuf, scount, sdtype,
rbuf, rcount, rdtype,
false, ucc_module, &req, coll_req));
Expand All @@ -124,7 +124,7 @@ int mca_coll_ucc_allgather_init(const void *sbuf, size_t scount, struct ompi_dat
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "allgather_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_allgather_init_common(sbuf, scount, sdtype,
rbuf, rcount, rdtype,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_allgatherv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ int mca_coll_ucc_iallgatherv(const void *sbuf, size_t scount,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc iallgatherv");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_allgatherv_init_common(sbuf, scount, sdtype,
rbuf, rcounts, rdisps, rdtype,
false, ucc_module, &req, coll_req));
Expand All @@ -131,7 +131,7 @@ int mca_coll_ucc_allgatherv_init(const void *sbuf, size_t scount, struct ompi_da
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "allgatherv_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_allgatherv_init_common(sbuf, scount, sdtype,
rbuf, rcounts, rdisps, rdtype,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_allreduce.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ int mca_coll_ucc_iallreduce(const void *sbuf, void *rbuf, size_t count,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc iallreduce");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_allreduce_init_common(sbuf, rbuf, count, dtype, op,
false, ucc_module, &req, coll_req));
COLL_UCC_POST_AND_CHECK(req);
Expand All @@ -116,7 +116,7 @@ int mca_coll_ucc_allreduce_init(const void *sbuf, void *rbuf, size_t count,
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "allreduce_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_allreduce_init_common(sbuf, rbuf, count, dtype, op,
true, ucc_module, &req, coll_req));
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_alltoall.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ int mca_coll_ucc_ialltoall(const void *sbuf, size_t scount, struct ompi_datatype
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ialltoall");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_alltoall_init_common(sbuf, scount, sdtype,
rbuf, rcount, rdtype,
false, ucc_module, &req, coll_req));
Expand All @@ -124,7 +124,7 @@ int mca_coll_ucc_alltoall_init(const void *sbuf, size_t scount, struct ompi_data
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "alltoall_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_alltoall_init_common(sbuf, scount, sdtype,
rbuf, rcount, rdtype,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_alltoallv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ int mca_coll_ucc_ialltoallv(const void *sbuf, ompi_count_array_t scounts,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ialltoallv");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_alltoallv_init_common(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype,
false, ucc_module, &req, coll_req));
Expand Down Expand Up @@ -134,7 +134,7 @@ int mca_coll_ucc_alltoallv_init(const void *sbuf, ompi_count_array_t scounts,
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "alltoallv_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_alltoallv_init_common(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_barrier.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ int mca_coll_ucc_ibarrier(struct ompi_communicator_t *comm,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ibarrier");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_barrier_init_common(false, ucc_module, &req, coll_req));
COLL_UCC_POST_AND_CHECK(req);
*request = &coll_req->super;
Expand All @@ -75,7 +75,7 @@ int mca_coll_ucc_barrier_init(struct ompi_communicator_t *comm, struct ompi_info
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "barrier_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_barrier_init_common(true, ucc_module, &req, coll_req));
*request = &coll_req->super;
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_bcast.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ int mca_coll_ucc_ibcast(void *buf, size_t count, struct ompi_datatype_t *dtype,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ibcast");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_bcast_init_common(buf, count, dtype, root,
false, ucc_module, &req, coll_req));
COLL_UCC_POST_AND_CHECK(req);
Expand All @@ -95,7 +95,7 @@ int mca_coll_ucc_bcast_init(void *buf, size_t count, struct ompi_datatype_t *dty
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "bcast_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_bcast_init_common(buf, count, dtype, root,
true, ucc_module, &req, coll_req));
Expand Down
6 changes: 4 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
} \
} while(0)

#define COLL_UCC_GET_REQ(_coll_req) do { \
#define COLL_UCC_GET_REQ(_coll_req, _comm) do { \
opal_free_list_item_t *item; \
item = opal_free_list_wait (&mca_coll_ucc_component.requests); \
if (OPAL_UNLIKELY(NULL == item)) { \
Expand All @@ -41,9 +41,10 @@
_coll_req->super.req_state = OMPI_REQUEST_ACTIVE; \
_coll_req->super.req_free = mca_coll_ucc_req_free; \
_coll_req->super.req_type = OMPI_REQUEST_COLL; \
_coll_req->super.req_mpi_object.comm = _comm; \
} while(0)

#define COLL_UCC_GET_REQ_PERSISTENT(_coll_req) \
#define COLL_UCC_GET_REQ_PERSISTENT(_coll_req, _comm) \
do { \
opal_free_list_item_t *item; \
item = opal_free_list_wait(&mca_coll_ucc_component.requests); \
Expand All @@ -59,6 +60,7 @@
_coll_req->super.req_free = mca_coll_ucc_req_free; \
_coll_req->super.req_start = mca_coll_ucc_req_start; \
_coll_req->super.req_type = OMPI_REQUEST_COLL; \
_coll_req->super.req_mpi_object.comm = _comm; \
_coll_req->ucc_req = NULL; \
} while (0)

Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_gather.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ int mca_coll_ucc_igather(const void *sbuf, size_t scount, struct ompi_datatype_t
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc igather");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_gather_init_common(sbuf, scount, sdtype, rbuf, rcount,
rdtype, root, false, ucc_module,
&req, coll_req));
Expand All @@ -142,7 +142,7 @@ int mca_coll_ucc_gather_init(const void *sbuf, size_t scount, struct ompi_dataty
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "gather_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_gather_init_common(sbuf, scount, sdtype, rbuf, rcount,
rdtype, root, true, ucc_module,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_gatherv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ int mca_coll_ucc_igatherv(const void *sbuf, size_t scount, struct ompi_datatype_
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc igatherv");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_gatherv_init_common(sbuf, scount, sdtype, rbuf, rcounts,
disps, rdtype, root, false, ucc_module,
&req, coll_req));
Expand All @@ -138,7 +138,7 @@ int mca_coll_ucc_gatherv_init(const void *sbuf, size_t scount, struct ompi_datat
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "gatherv_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_gatherv_init_common(sbuf, scount, sdtype, rbuf, rcounts,
disps, rdtype, root, true, ucc_module,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_reduce.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ int mca_coll_ucc_ireduce(const void *sbuf, void* rbuf, size_t count,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ireduce");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_reduce_init_common(sbuf, rbuf, count, dtype, op, root,
false, ucc_module, &req, coll_req));
COLL_UCC_POST_AND_CHECK(req);
Expand All @@ -119,7 +119,7 @@ int mca_coll_ucc_reduce_init(const void *sbuf, void *rbuf, size_t count,
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "reduce_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_reduce_init_common(sbuf, rbuf, count, dtype, op, root,
true, ucc_module, &req, coll_req));
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_reduce_scatter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ int mca_coll_ucc_ireduce_scatter(const void *sbuf, void *rbuf, ompi_count_array_
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ireduce_scatter");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_init_common(sbuf, rbuf, rcounts, dtype,
op, false, ucc_module, &req, coll_req));
COLL_UCC_POST_AND_CHECK(req);
Expand All @@ -134,7 +134,7 @@ int mca_coll_ucc_reduce_scatter_init(const void *sbuf, void *rbuf, ompi_count_ar
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "reduce_scatter_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_init_common(sbuf, rbuf, rcounts, dtype,
op, true, ucc_module, &req, coll_req));
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_reduce_scatter_block.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ int mca_coll_ucc_ireduce_scatter_block(const void *sbuf, void *rbuf, size_t rcou
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc ireduce_scatter_block");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_block_init_common(sbuf, rbuf, rcount,
dtype, op, false, ucc_module,
&req, coll_req));
Expand All @@ -132,7 +132,7 @@ int mca_coll_ucc_reduce_scatter_block_init(const void *sbuf, void *rbuf, size_t
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "reduce_scatter_block_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_reduce_scatter_block_init_common(sbuf, rbuf, rcount,
dtype, op, true, ucc_module,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_scatter.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ int mca_coll_ucc_iscatter(const void *sbuf, size_t scount,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc iscatter");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_scatter_init_common(sbuf, scount, sdtype, rbuf, rcount,
rdtype, root, false, ucc_module, &req,
coll_req));
Expand All @@ -146,7 +146,7 @@ int mca_coll_ucc_scatter_init(const void *sbuf, size_t scount, struct ompi_datat
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "scatter_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_scatter_init_common(sbuf, scount, sdtype, rbuf, rcount,
rdtype, root, true, ucc_module, &req,
Expand Down
4 changes: 2 additions & 2 deletions ompi/mca/coll/ucc/coll_ucc_scatterv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ int mca_coll_ucc_iscatterv(const void *sbuf, ompi_count_array_t scounts,
mca_coll_ucc_req_t *coll_req = NULL;

UCC_VERBOSE(3, "running ucc iscatterv");
COLL_UCC_GET_REQ(coll_req);
COLL_UCC_GET_REQ(coll_req, comm);
COLL_UCC_CHECK(mca_coll_ucc_scatterv_init_common(sbuf, scounts, disps, sdtype,
rbuf, rcount, rdtype, root,
false, ucc_module, &req, coll_req));
Expand All @@ -140,7 +140,7 @@ int mca_coll_ucc_scatterv_init(const void *sbuf, ompi_count_array_t scounts,
ucc_coll_req_h req;
mca_coll_ucc_req_t *coll_req = NULL;

COLL_UCC_GET_REQ_PERSISTENT(coll_req);
COLL_UCC_GET_REQ_PERSISTENT(coll_req, comm);
UCC_VERBOSE(3, "scatterv_init init %p", coll_req);
COLL_UCC_CHECK(mca_coll_ucc_scatterv_init_common(sbuf, scounts, disps, sdtype,
rbuf, rcount, rdtype, root,
Expand Down