Skip to content

Commit fb93dc6

Browse files
Yogaraj-AlamendaYogaraj-Alamenda
committed
Refactor qat_sw signalling, timeout & fix err scenarios.
- Changes to polling thread signalling to send signal only on the 8th request instead of every request. - Add option to change polling timeout(nsecs) with the build configure eg: --with-cc_opt="-DQAT_SW_POLL_TIMEOUT_NSEC=100000" - Add freelist cleanup in error conditions. Signed-off-by: Yogaraj Alamenda <yogarajx.alamenda@intel.com>
1 parent eec2054 commit fb93dc6

File tree

10 files changed

+324
-336
lines changed

10 files changed

+324
-336
lines changed

‎qat_fork.c‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
#include <stdlib.h>
5858
#include <string.h>
5959
#include <pthread.h>
60+
#include <signal.h>
6061
#include <fcntl.h>
6162

6263
/* Local Includes */
@@ -77,6 +78,13 @@
7778
# endif
7879
# include "cpa.h"
7980
# include "cpa_types.h"
81+
82+
# ifndef __FreeBSD__
83+
typedef cpu_set_t qat_cpuset;
84+
# else
85+
# include <pthread_np.h>
86+
typedef cpuset_t qat_cpuset;
87+
# endif
8088
#endif
8189

8290
void engine_init_child_at_fork_handler(void)
@@ -115,13 +123,72 @@ void engine_finish_before_fork_handler(void)
115123
multibuff_keep_polling = 1;
116124
}
117125

126+
int qat_create_thread(pthread_t *pThreadId, const pthread_attr_t *attr,
127+
void *(*start_func) (void *), void *pArg)
128+
{
129+
return pthread_create(pThreadId, attr, start_func,(void *)pArg);
130+
}
131+
132+
int qat_join_thread(pthread_t threadId, void **retval)
133+
{
134+
return pthread_join(threadId, retval);
135+
}
136+
137+
int qat_kill_thread(pthread_t threadId, int sig)
138+
{
139+
return pthread_kill(threadId, sig);
140+
}
141+
142+
int qat_setspecific_thread(pthread_key_t key, const void *value)
143+
{
144+
return pthread_setspecific(key, value);
145+
}
146+
147+
void *qat_getspecific_thread(pthread_key_t key)
148+
{
149+
return pthread_getspecific(key);
150+
}
151+
118152
#ifdef QAT_HW
153+
int qat_adjust_thread_affinity(pthread_t threadptr)
154+
{
155+
# ifdef QAT_POLL_CORE_AFFINITY
156+
int coreID = 0;
157+
int sts = 1;
158+
qat_cpuset cpuset;
159+
CPU_ZERO(&cpuset);
160+
CPU_SET(coreID, &cpuset);
161+
162+
sts = pthread_setaffinity_np(threadptr, sizeof(qat_cpuset), &cpuset);
163+
if (sts != 0) {
164+
WARN("pthread_setaffinity_np error, status = %d\n", sts);
165+
QATerr(QAT_F_QAT_ADJUST_THREAD_AFFINITY, QAT_R_PTHREAD_SETAFFINITY_FAILURE);
166+
return 0;
167+
}
168+
sts = pthread_getaffinity_np(threadptr, sizeof(qat_cpuset), &cpuset);
169+
if (sts != 0) {
170+
WARN("pthread_getaffinity_np error, status = %d\n", sts);
171+
QATerr(QAT_F_QAT_ADJUST_THREAD_AFFINITY, QAT_R_PTHREAD_GETAFFINITY_FAILURE);
172+
return 0;
173+
}
174+
175+
if (CPU_ISSET(coreID, &cpuset)) {
176+
DEBUG("Polling thread assigned on CPU core %d\n", coreID);
177+
}
178+
# endif
179+
return 1;
180+
}
181+
182+
int qat_fcntl(int fd, int cmd, int arg)
183+
{
184+
return fcntl(fd, cmd, arg);
185+
}
186+
119187
int qat_set_instance_for_thread(long instanceNum)
120188
{
121189
thread_local_variables_t *tlv = NULL;
122190
tlv = qat_check_create_local_variables();
123-
if (NULL == tlv ||
124-
0 == qat_num_instances ||
191+
if (NULL == tlv || 0 == qat_num_instances ||
125192
instanceNum < 0) {
126193
WARN("could not create local variables or no instances available\n");
127194
QATerr(QAT_F_QAT_SET_INSTANCE_FOR_THREAD, QAT_R_SET_INSTANCE_FAILURE);

‎qat_fork.h‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@
4646
#ifndef QAT_FORK_H
4747
# define QAT_FORK_H
4848

49-
# include "e_qat.h"
49+
# include "e_qat.h"
50+
51+
int qat_fcntl(int fd, int cmd, int arg);
5052

5153
/******************************************************************************
5254
* function:
@@ -68,6 +70,82 @@ void engine_init_child_at_fork_handler(void);
6870
******************************************************************************/
6971
void engine_finish_before_fork_handler(void);
7072

73+
/******************************************************************************
74+
* function:
75+
* int qat_create_thread(pthread_t *pThreadId,
76+
* const pthread_attr_t *attr,
77+
* void *(*start_func) (void *), void *pArg)
78+
*
79+
* @param pThreadId [OUT] - Pointer to Thread ID
80+
* @param start_func [IN] - Pointer to Thread Start routine
81+
* @param attr [IN] - Pointer to Thread attributes
82+
* @param pArg [IN] - Arguments to start routine
83+
*
84+
* description:
85+
* Wrapper function for pthread_create
86+
******************************************************************************/
87+
int qat_create_thread(pthread_t *pThreadId, const pthread_attr_t *attr,
88+
void *(*start_func) (void *), void *pArg);
89+
90+
/******************************************************************************
91+
* function:
92+
* int qat_join_thread(pthread_t threadId, void **retval)
93+
*
94+
* @param pThreadId [IN] - Thread ID of the created thread
95+
* @param retval [OUT] - Pointer that contains thread's exit status
96+
*
97+
* description:
98+
* Wrapper function for pthread_create
99+
******************************************************************************/
100+
int qat_join_thread(pthread_t threadId, void **retval);
101+
102+
/******************************************************************************
103+
* function:
104+
* int qat_kill_thread(pthread_t threadId, int sig)
105+
*
106+
* @param pThreadId [IN] - Thread ID of the created thread
107+
* @param sig [IN] - Signal number
108+
*
109+
* description:
110+
* Wrapper function for pthread_kill
111+
******************************************************************************/
112+
int qat_kill_thread(pthread_t threadId, int sig);
113+
114+
/******************************************************************************
115+
* function:
116+
* int qat_setspecific_thread(pthread_key_t key, const void *value)
117+
*
118+
* @param key [IN] - key obtained from pthread_key_create()
119+
* @param value [IN] - Thread specific value
120+
*
121+
* description:
122+
* Wrapper function for pthread_setspecific
123+
******************************************************************************/
124+
int qat_setspecific_thread(pthread_key_t key, const void *value);
125+
126+
/******************************************************************************
127+
* function:
128+
* int qat_getspecific_thread(pthread_key_t key)
129+
*
130+
* @param key [IN] - key obtained from pthread_key_create()
131+
*
132+
* description:
133+
* Wrapper function for pthread_getspecific
134+
******************************************************************************/
135+
void *qat_getspecific_thread(pthread_key_t key);
136+
137+
/******************************************************************************
138+
* function:
139+
* int qat_adjust_thread_affinity(pthread_t threadptr);
140+
*
141+
* @param threadptr[IN ] - Thread ID
142+
*
143+
* description:
144+
* Sets the CPU affinity mask using pthread_setaffinity_np
145+
* and returns the CPU affinity mask using pthread_getaffinity_np
146+
******************************************************************************/
147+
int qat_adjust_thread_affinity(pthread_t threadptr);
148+
71149
/******************************************************************************
72150
* function:
73151
* int qat_set_instance_for_thread(long instanceNum)

‎qat_hw_polling.c‎

Lines changed: 0 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,8 @@ ENGINE_EPOLL_ST eng_poll_st[QAT_MAX_CRYPTO_INSTANCES] = {{ -1 }};
9191
#endif
9292
int internal_efd = 0;
9393
#ifndef __FreeBSD__
94-
typedef cpu_set_t qat_cpuset;
9594
clock_t clock_id = CLOCK_MONOTONIC_RAW;
9695
#else
97-
# include <pthread_np.h>
98-
typedef cpuset_t qat_cpuset;
9996
clock_t clock_id = CLOCK_MONOTONIC_PRECISE;
10097
#endif
10198

@@ -115,66 +112,6 @@ int getEnableInlinePolling()
115112
return enable_inline_polling;
116113
}
117114

118-
int qat_create_thread(pthread_t *pThreadId, const pthread_attr_t *attr,
119-
void *(*start_func) (void *), void *pArg)
120-
{
121-
return pthread_create(pThreadId, attr, start_func,(void *)pArg);
122-
}
123-
124-
int qat_join_thread(pthread_t threadId, void **retval)
125-
{
126-
return pthread_join(threadId, retval);
127-
}
128-
129-
int qat_kill_thread(pthread_t threadId, int sig)
130-
{
131-
return pthread_kill(threadId, sig);
132-
}
133-
134-
int qat_setspecific_thread(pthread_key_t key, const void *value)
135-
{
136-
return pthread_setspecific(key, value);
137-
}
138-
139-
void *qat_getspecific_thread(pthread_key_t key)
140-
{
141-
return pthread_getspecific(key);
142-
}
143-
144-
int qat_adjust_thread_affinity(pthread_t threadptr)
145-
{
146-
#ifdef QAT_POLL_CORE_AFFINITY
147-
int coreID = 0;
148-
int sts = 1;
149-
qat_cpuset cpuset;
150-
CPU_ZERO(&cpuset);
151-
CPU_SET(coreID, &cpuset);
152-
153-
sts = pthread_setaffinity_np(threadptr, sizeof(qat_cpuset), &cpuset);
154-
if (sts != 0) {
155-
WARN("pthread_setaffinity_np error, status = %d\n", sts);
156-
QATerr(QAT_F_QAT_ADJUST_THREAD_AFFINITY, QAT_R_PTHREAD_SETAFFINITY_FAILURE);
157-
return 0;
158-
}
159-
sts = pthread_getaffinity_np(threadptr, sizeof(qat_cpuset), &cpuset);
160-
if (sts != 0) {
161-
WARN("pthread_getaffinity_np error, status = %d\n", sts);
162-
QATerr(QAT_F_QAT_ADJUST_THREAD_AFFINITY, QAT_R_PTHREAD_GETAFFINITY_FAILURE);
163-
return 0;
164-
}
165-
166-
if (CPU_ISSET(coreID, &cpuset)) {
167-
DEBUG("Polling thread assigned on CPU core %d\n", coreID);
168-
}
169-
#endif
170-
return 1;
171-
}
172-
173-
int qat_fcntl(int fd, int cmd, int arg)
174-
{
175-
return fcntl(fd, cmd, arg);
176-
}
177-
178115
static void qat_poll_heartbeat_timer_expiry(struct timespec *previous_time)
179116
{
180117
struct timespec current_time = { 0 };

‎qat_hw_polling.h‎

Lines changed: 5 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,11 @@
4646
#ifndef QAT_HW_POLLING_H
4747
# define QAT_HW_POLLING_H
4848

49-
# ifndef QAT_SW
50-
# include "cpa.h"
51-
# include "cpa_types.h"
52-
# include "e_qat.h"
53-
# else
54-
# include "e_qat.h"
55-
# endif
49+
# include "cpa.h"
50+
# include "cpa_types.h"
51+
52+
# include "e_qat.h"
53+
# include "qat_fork.h"
5654

5755
# ifndef __FreeBSD__
5856
# include <sys/epoll.h>
@@ -74,81 +72,6 @@ extern int internal_efd;
7472
int getQatMsgRetryCount();
7573
useconds_t getQatPollInterval();
7674
int getEnableInlinePolling();
77-
/******************************************************************************
78-
* function:
79-
* int qat_create_thread(pthread_t *pThreadId,
80-
* const pthread_attr_t *attr,
81-
* void *(*start_func) (void *), void *pArg)
82-
*
83-
* @param pThreadId [OUT] - Pointer to Thread ID
84-
* @param start_func [IN] - Pointer to Thread Start routine
85-
* @param attr [IN] - Pointer to Thread attributes
86-
* @param pArg [IN] - Arguments to start routine
87-
*
88-
* description:
89-
* Wrapper function for pthread_create
90-
******************************************************************************/
91-
int qat_create_thread(pthread_t *pThreadId, const pthread_attr_t *attr,
92-
void *(*start_func) (void *), void *pArg);
93-
94-
/******************************************************************************
95-
* function:
96-
* int qat_join_thread(pthread_t threadId, void **retval)
97-
*
98-
* @param pThreadId [IN] - Thread ID of the created thread
99-
* @param retval [OUT] - Pointer that contains thread's exit status
100-
*
101-
* description:
102-
* Wrapper function for pthread_create
103-
******************************************************************************/
104-
int qat_join_thread(pthread_t threadId, void **retval);
105-
106-
/******************************************************************************
107-
* function:
108-
* int qat_kill_thread(pthread_t threadId, int sig)
109-
*
110-
* @param pThreadId [IN] - Thread ID of the created thread
111-
* @param sig [IN] - Signal number
112-
*
113-
* description:
114-
* Wrapper function for pthread_kill
115-
******************************************************************************/
116-
int qat_kill_thread(pthread_t threadId, int sig);
117-
118-
/******************************************************************************
119-
* function:
120-
* int qat_setspecific_thread(pthread_key_t key, const void *value)
121-
*
122-
* @param key [IN] - key obtained from pthread_key_create()
123-
* @param value [IN] - Thread specific value
124-
*
125-
* description:
126-
* Wrapper function for pthread_setspecific
127-
******************************************************************************/
128-
int qat_setspecific_thread(pthread_key_t key, const void *value);
129-
130-
/******************************************************************************
131-
* function:
132-
* int qat_getspecific_thread(pthread_key_t key)
133-
*
134-
* @param key [IN] - key obtained from pthread_key_create()
135-
*
136-
* description:
137-
* Wrapper function for pthread_getspecific
138-
******************************************************************************/
139-
void *qat_getspecific_thread(pthread_key_t key);
140-
141-
/******************************************************************************
142-
* function:
143-
* int qat_adjust_thread_affinity(pthread_t threadptr);
144-
*
145-
* @param threadptr[IN ] - Thread ID
146-
*
147-
* description:
148-
* Sets the CPU affinity mask using pthread_setaffinity_np
149-
* and returns the CPU affinity mask using pthread_getaffinity_np
150-
******************************************************************************/
151-
int qat_adjust_thread_affinity(pthread_t threadptr);
15275

15376
/******************************************************************************
15477
* function:
@@ -167,8 +90,6 @@ int qat_adjust_thread_affinity(pthread_t threadptr);
16790
******************************************************************************/
16891
void *qat_timer_poll_func(void *ih);
16992

170-
int qat_fcntl(int fd, int cmd, int arg);
171-
17293
# ifndef __FreeBSD__
17394
void *event_poll_func(void *ih);
17495
# endif

0 commit comments

Comments
 (0)