From 4c879170296174bde05cd1c643dac16594edee77 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:30:10 +0200 Subject: padata: Check for valid padata instance on start This patch introduces the PADATA_INVALID flag which is checked on padata start. This will be used to mark a padata instance as invalid, if the padata cpumask does not intersect with the active cpumask. we change padata_start to return an error if the PADATA_INVALID is set. Also we adapt the only padata user, pcrypt to this change. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 8d8406246ee..e4c17f9b7c9 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -126,6 +126,7 @@ struct padata_instance { u8 flags; #define PADATA_INIT 1 #define PADATA_RESET 2 +#define PADATA_INVALID 4 }; extern struct padata_instance *padata_alloc(const struct cpumask *cpumask, @@ -138,6 +139,6 @@ extern int padata_set_cpumask(struct padata_instance *pinst, cpumask_var_t cpumask); extern int padata_add_cpu(struct padata_instance *pinst, int cpu); extern int padata_remove_cpu(struct padata_instance *pinst, int cpu); -extern void padata_start(struct padata_instance *pinst); +extern int padata_start(struct padata_instance *pinst); extern void padata_stop(struct padata_instance *pinst); #endif -- cgit v1.2.3-18-g5258 From 5f1a8c1bc724498ff32acbd59ed5263275676b9d Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Wed, 7 Jul 2010 15:32:39 +0200 Subject: padata: simplify serialization mechanism We count the number of processed objects on a percpu basis, so we need to go through all the percpu reorder queues to calculate the sequence number of the next object that needs serialization. This patch changes this to count the number of processed objects global. So we can calculate the sequence number and the percpu reorder queue of the next object that needs serialization without searching through the percpu reorder queues. This avoids some accesses to memory of foreign cpus. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index e4c17f9b7c9..8844b851191 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -67,7 +67,6 @@ struct padata_list { * @pwork: work struct for parallelization. * @swork: work struct for serialization. * @pd: Backpointer to the internal control structure. - * @num_obj: Number of objects that are processed by this cpu. * @cpu_index: Index of the cpu. */ struct padata_queue { @@ -77,7 +76,6 @@ struct padata_queue { struct work_struct pwork; struct work_struct swork; struct parallel_data *pd; - atomic_t num_obj; int cpu_index; }; @@ -93,6 +91,7 @@ struct padata_queue { * @max_seq_nr: Maximal used sequence number. * @cpumask: cpumask in use. * @lock: Reorder lock. + * @processed: Number of already processed objects. * @timer: Reorder timer. */ struct parallel_data { @@ -103,7 +102,8 @@ struct parallel_data { atomic_t refcnt; unsigned int max_seq_nr; cpumask_var_t cpumask; - spinlock_t lock; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; struct timer_list timer; }; -- cgit v1.2.3-18-g5258 From e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8 Mon Sep 17 00:00:00 2001 From: Dan Kruchinin Date: Wed, 14 Jul 2010 14:31:57 +0400 Subject: padata: Make two separate cpumasks The aim of this patch is to make two separate cpumasks for padata parallel and serial workers respectively. It allows user to make more thin and sophisticated configurations of padata framework. For example user may bind parallel and serial workers to non-intersecting CPU groups to gain better performance. Also each padata instance has notifiers chain for its cpumasks now. If either parallel or serial or both masks were changed all interested subsystems will get notification about that. It's especially useful if padata user uses algorithm for callback CPU selection according to serial cpumask. Signed-off-by: Dan Kruchinin Signed-off-by: Herbert Xu --- include/linux/padata.h | 116 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 36 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 8844b851191..621e7736690 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -25,6 +25,10 @@ #include #include #include +#include + +#define PADATA_CPU_SERIAL 0x01 +#define PADATA_CPU_PARALLEL 0x02 /** * struct padata_priv - Embedded to the users data structure. @@ -59,7 +63,20 @@ struct padata_list { }; /** - * struct padata_queue - The percpu padata queues. +* struct padata_serial_queue - The percpu padata serial queue +* +* @serial: List to wait for serialization after reordering. +* @work: work struct for serialization. +* @pd: Backpointer to the internal control structure. +*/ +struct padata_serial_queue { + struct padata_list serial; + struct work_struct work; + struct parallel_data *pd; +}; + +/** + * struct padata_parallel_queue - The percpu padata parallel queue * * @parallel: List to wait for parallelization. * @reorder: List to wait for reordering after parallel processing. @@ -67,44 +84,52 @@ struct padata_list { * @pwork: work struct for parallelization. * @swork: work struct for serialization. * @pd: Backpointer to the internal control structure. + * @work: work struct for parallelization. + * @num_obj: Number of objects that are processed by this cpu. * @cpu_index: Index of the cpu. */ -struct padata_queue { - struct padata_list parallel; - struct padata_list reorder; - struct padata_list serial; - struct work_struct pwork; - struct work_struct swork; - struct parallel_data *pd; - int cpu_index; +struct padata_parallel_queue { + struct padata_list parallel; + struct padata_list reorder; + struct parallel_data *pd; + struct work_struct work; + atomic_t num_obj; + int cpu_index; }; + /** * struct parallel_data - Internal control structure, covers everything * that depends on the cpumask in use. * * @pinst: padata instance. - * @queue: percpu padata queues. + * @pqueue: percpu padata queues used for parallelization. + * @squeue: percpu padata queues used for serialuzation. * @seq_nr: The sequence number that will be attached to the next object. * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. - * @cpumask: cpumask in use. + * @cpumask: Contains two cpumasks: pcpu and cbcpu for + * parallel and serial workers respectively. * @lock: Reorder lock. * @processed: Number of already processed objects. * @timer: Reorder timer. */ struct parallel_data { - struct padata_instance *pinst; - struct padata_queue *queue; - atomic_t seq_nr; - atomic_t reorder_objects; - atomic_t refcnt; - unsigned int max_seq_nr; - cpumask_var_t cpumask; - spinlock_t lock ____cacheline_aligned; - unsigned int processed; - struct timer_list timer; + struct padata_instance *pinst; + struct padata_parallel_queue *pqueue; + struct padata_serial_queue *squeue; + atomic_t seq_nr; + atomic_t reorder_objects; + atomic_t refcnt; + unsigned int max_seq_nr; + struct { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; + } cpumask; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; + struct timer_list timer; }; /** @@ -113,32 +138,51 @@ struct parallel_data { * @cpu_notifier: cpu hotplug notifier. * @wq: The workqueue in use. * @pd: The internal control structure. - * @cpumask: User supplied cpumask. + * @cpumask: User supplied cpumask. Contains two cpumasks: pcpu and + * cbcpu for parallel and serial works respectivly. + * @cpumask_change_notifier: Notifiers chain for user-defined notify + * callbacks that will be called when either @pcpu or @cbcpu + * or both cpumasks change. * @lock: padata instance lock. * @flags: padata flags. */ struct padata_instance { - struct notifier_block cpu_notifier; - struct workqueue_struct *wq; - struct parallel_data *pd; - cpumask_var_t cpumask; - struct mutex lock; - u8 flags; -#define PADATA_INIT 1 -#define PADATA_RESET 2 -#define PADATA_INVALID 4 + struct notifier_block cpu_notifier; + struct workqueue_struct *wq; + struct parallel_data *pd; + struct { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; + } cpumask; + struct blocking_notifier_head cpumask_change_notifier; + struct mutex lock; + u8 flags; +#define PADATA_INIT 1 +#define PADATA_RESET 2 +#define PADATA_INVALID 4 }; -extern struct padata_instance *padata_alloc(const struct cpumask *cpumask, - struct workqueue_struct *wq); +extern struct padata_instance *padata_alloc(struct workqueue_struct *wq); +extern struct padata_instance *__padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); extern void padata_do_serial(struct padata_priv *padata); -extern int padata_set_cpumask(struct padata_instance *pinst, +extern int padata_get_cpumask(struct padata_instance *pinst, + int cpumask_type, struct cpumask *out_mask); +extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); -extern int padata_add_cpu(struct padata_instance *pinst, int cpu); -extern int padata_remove_cpu(struct padata_instance *pinst, int cpu); +extern int __padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask); +extern int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); +extern int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_start(struct padata_instance *pinst); extern void padata_stop(struct padata_instance *pinst); +extern int padata_register_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); +extern int padata_unregister_cpumask_notifier(struct padata_instance *pinst, + struct notifier_block *nblock); #endif -- cgit v1.2.3-18-g5258 From 5e017dc3f8bc9e4a28983666e6bc00114a2018bb Mon Sep 17 00:00:00 2001 From: Dan Kruchinin Date: Wed, 14 Jul 2010 14:33:08 +0400 Subject: padata: Added sysfs primitives to padata subsystem Added sysfs primitives to padata subsystem. Now API user may embedded kobject each padata instance contains into any sysfs hierarchy. For now padata sysfs interface provides only two objects: serial_cpumask [RW] - cpumask for serial workers parallel_cpumask [RW] - cpumask for parallel workers Signed-off-by: Dan Kruchinin Signed-off-by: Herbert Xu --- include/linux/padata.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 621e7736690..293ad46ffce 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -26,6 +26,7 @@ #include #include #include +#include #define PADATA_CPU_SERIAL 0x01 #define PADATA_CPU_PARALLEL 0x02 @@ -142,7 +143,8 @@ struct parallel_data { * cbcpu for parallel and serial works respectivly. * @cpumask_change_notifier: Notifiers chain for user-defined notify * callbacks that will be called when either @pcpu or @cbcpu - * or both cpumasks change. + * or both cpumasks change. + * @kobj: padata instance kernel object. * @lock: padata instance lock. * @flags: padata flags. */ @@ -155,6 +157,7 @@ struct padata_instance { cpumask_var_t cbcpu; } cpumask; struct blocking_notifier_head cpumask_change_notifier; + struct kobject kobj; struct mutex lock; u8 flags; #define PADATA_INIT 1 -- cgit v1.2.3-18-g5258 From e6cc11707661770ca2bd4db4b0256d28f48e7541 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:14:28 +0200 Subject: padata: Rename padata_alloc functions We rename padata_alloc to padata_alloc_possible because this function allocates a padata_instance and uses the cpu_possible mask for parallel and serial workers. Also we rename __padata_alloc to padata_alloc to avoid to export underlined functions. Underlined functions are considered to be private to padata. Users are updated accordingly. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 293ad46ffce..71dfc9d1f85 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -165,10 +165,11 @@ struct padata_instance { #define PADATA_INVALID 4 }; -extern struct padata_instance *padata_alloc(struct workqueue_struct *wq); -extern struct padata_instance *__padata_alloc(struct workqueue_struct *wq, - const struct cpumask *pcpumask, - const struct cpumask *cbcpumask); +extern struct padata_instance *padata_alloc_possible( + struct workqueue_struct *wq); +extern struct padata_instance *padata_alloc(struct workqueue_struct *wq, + const struct cpumask *pcpumask, + const struct cpumask *cbcpumask); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); -- cgit v1.2.3-18-g5258 From 65ff577e6b6e482ee9de3569e058edebdc02f069 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:15:06 +0200 Subject: padata: Rearrange set_cpumask functions padata_set_cpumask needs to be protected by a lock. We make __padata_set_cpumasks unlocked and static. So this function can be used by the exported and locked padata_set_cpumask and padata_set_cpumasks functions. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 71dfc9d1f85..bb0fc5dd0bb 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -178,9 +178,9 @@ extern int padata_get_cpumask(struct padata_instance *pinst, int cpumask_type, struct cpumask *out_mask); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); -extern int __padata_set_cpumasks(struct padata_instance *pinst, - cpumask_var_t pcpumask, - cpumask_var_t cbcpumask); +extern int padata_set_cpumasks(struct padata_instance *pinst, + cpumask_var_t pcpumask, + cpumask_var_t cbcpumask); extern int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask); extern int padata_start(struct padata_instance *pinst); -- cgit v1.2.3-18-g5258 From c635696c7c0fbc720698dbec34bb83e53df6a967 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:15:50 +0200 Subject: padata: Pass the padata cpumasks to the cpumask_change_notifier chain We pass a pointer to the new padata cpumasks to the cpumask_change_notifier chain. So users can access the cpumasks without the need of an extra padata_get_cpumask function. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index bb0fc5dd0bb..43db792f44d 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -98,6 +98,16 @@ struct padata_parallel_queue { int cpu_index; }; +/** + * struct padata_cpumask - The cpumasks for the parallel/serial workers + * + * @pcpu: cpumask for the parallel workers. + * @cbcpu: cpumask for the serial (callback) workers. + */ +struct padata_cpumask { + cpumask_var_t pcpu; + cpumask_var_t cbcpu; +}; /** * struct parallel_data - Internal control structure, covers everything @@ -110,8 +120,7 @@ struct padata_parallel_queue { * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. - * @cpumask: Contains two cpumasks: pcpu and cbcpu for - * parallel and serial workers respectively. + * @cpumask: The cpumasks in use for parallel and serial workers. * @lock: Reorder lock. * @processed: Number of already processed objects. * @timer: Reorder timer. @@ -120,17 +129,14 @@ struct parallel_data { struct padata_instance *pinst; struct padata_parallel_queue *pqueue; struct padata_serial_queue *squeue; - atomic_t seq_nr; - atomic_t reorder_objects; - atomic_t refcnt; - unsigned int max_seq_nr; - struct { - cpumask_var_t pcpu; - cpumask_var_t cbcpu; - } cpumask; - spinlock_t lock ____cacheline_aligned; - unsigned int processed; - struct timer_list timer; + atomic_t seq_nr; + atomic_t reorder_objects; + atomic_t refcnt; + unsigned int max_seq_nr; + struct padata_cpumask cpumask; + spinlock_t lock ____cacheline_aligned; + unsigned int processed; + struct timer_list timer; }; /** @@ -139,8 +145,7 @@ struct parallel_data { * @cpu_notifier: cpu hotplug notifier. * @wq: The workqueue in use. * @pd: The internal control structure. - * @cpumask: User supplied cpumask. Contains two cpumasks: pcpu and - * cbcpu for parallel and serial works respectivly. + * @cpumask: User supplied cpumasks for parallel and serial works. * @cpumask_change_notifier: Notifiers chain for user-defined notify * callbacks that will be called when either @pcpu or @cbcpu * or both cpumasks change. @@ -152,10 +157,7 @@ struct padata_instance { struct notifier_block cpu_notifier; struct workqueue_struct *wq; struct parallel_data *pd; - struct { - cpumask_var_t pcpu; - cpumask_var_t cbcpu; - } cpumask; + struct padata_cpumask cpumask; struct blocking_notifier_head cpumask_change_notifier; struct kobject kobj; struct mutex lock; -- cgit v1.2.3-18-g5258 From 0500e9b3f11ce84fc6ee48a3e29909145e58ba48 Mon Sep 17 00:00:00 2001 From: Steffen Klassert Date: Tue, 27 Jul 2010 07:19:27 +0200 Subject: padata: Remove padata_get_cpumask A function that copies the padata cpumasks to a user buffer is a bit error prone. The cpumask can change any time so we can't be sure to have the right cpumask when using this function. A user who is interested in the padata cpumasks should register to the padata cpumask notifier chain instead. Users of padata_get_cpumask are already updated, so we can remove it. Signed-off-by: Steffen Klassert Signed-off-by: Herbert Xu --- include/linux/padata.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 43db792f44d..bdcd1e9eace 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -176,8 +176,6 @@ extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); extern void padata_do_serial(struct padata_priv *padata); -extern int padata_get_cpumask(struct padata_instance *pinst, - int cpumask_type, struct cpumask *out_mask); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); extern int padata_set_cpumasks(struct padata_instance *pinst, -- cgit v1.2.3-18-g5258