diff options
Diffstat (limited to 'fs/lockd')
-rw-r--r-- | fs/lockd/grace.c | 16 | ||||
-rw-r--r-- | fs/lockd/host.c | 92 | ||||
-rw-r--r-- | fs/lockd/netns.h | 7 | ||||
-rw-r--r-- | fs/lockd/svc.c | 43 | ||||
-rw-r--r-- | fs/lockd/svc4proc.c | 13 | ||||
-rw-r--r-- | fs/lockd/svclock.c | 16 | ||||
-rw-r--r-- | fs/lockd/svcproc.c | 15 | ||||
-rw-r--r-- | fs/lockd/svcsubs.c | 19 |
8 files changed, 142 insertions, 79 deletions
diff --git a/fs/lockd/grace.c b/fs/lockd/grace.c index 183cc1f0af1..6d1ee7204c8 100644 --- a/fs/lockd/grace.c +++ b/fs/lockd/grace.c @@ -4,8 +4,10 @@ #include <linux/module.h> #include <linux/lockd/bind.h> +#include <net/net_namespace.h> + +#include "netns.h" -static LIST_HEAD(grace_list); static DEFINE_SPINLOCK(grace_lock); /** @@ -19,10 +21,12 @@ static DEFINE_SPINLOCK(grace_lock); * * This function is called to start a grace period. */ -void locks_start_grace(struct lock_manager *lm) +void locks_start_grace(struct net *net, struct lock_manager *lm) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + spin_lock(&grace_lock); - list_add(&lm->list, &grace_list); + list_add(&lm->list, &ln->grace_list); spin_unlock(&grace_lock); } EXPORT_SYMBOL_GPL(locks_start_grace); @@ -52,8 +56,10 @@ EXPORT_SYMBOL_GPL(locks_end_grace); * to answer ordinary lock requests, and when they should accept only * lock reclaims. */ -int locks_in_grace(void) +int locks_in_grace(struct net *net) { - return !list_empty(&grace_list); + struct lockd_net *ln = net_generic(net, lockd_net_id); + + return !list_empty(&ln->grace_list); } EXPORT_SYMBOL_GPL(locks_in_grace); diff --git a/fs/lockd/host.c b/fs/lockd/host.c index eb75ca7c2d6..f9b22e58f78 100644 --- a/fs/lockd/host.c +++ b/fs/lockd/host.c @@ -21,6 +21,8 @@ #include <net/ipv6.h> +#include "netns.h" + #define NLMDBG_FACILITY NLMDBG_HOSTCACHE #define NLM_HOST_NRHASH 32 #define NLM_HOST_REBIND (60 * HZ) @@ -41,11 +43,10 @@ static struct hlist_head nlm_client_hosts[NLM_HOST_NRHASH]; hlist_for_each_entry_safe((host), (pos), (next), \ (chain), h_hash) -static unsigned long next_gc; static unsigned long nrhosts; static DEFINE_MUTEX(nlm_host_mutex); -static void nlm_gc_hosts(void); +static void nlm_gc_hosts(struct net *net); struct nlm_lookup_host_info { const int server; /* search for server|client */ @@ -172,6 +173,7 @@ out: static void nlm_destroy_host_locked(struct nlm_host *host) { struct rpc_clnt *clnt; + struct lockd_net *ln = net_generic(host->net, lockd_net_id); dprintk("lockd: destroy host %s\n", host->h_name); @@ -188,6 +190,7 @@ static void nlm_destroy_host_locked(struct nlm_host *host) rpc_shutdown_client(clnt); kfree(host); + ln->nrhosts--; nrhosts--; } @@ -228,6 +231,7 @@ struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap, struct hlist_node *pos; struct nlm_host *host; struct nsm_handle *nsm = NULL; + struct lockd_net *ln = net_generic(net, lockd_net_id); dprintk("lockd: %s(host='%s', vers=%u, proto=%s)\n", __func__, (hostname ? hostname : "<none>"), version, @@ -262,6 +266,7 @@ struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap, goto out; hlist_add_head(&host->h_hash, chain); + ln->nrhosts++; nrhosts++; dprintk("lockd: %s created host %s (%s)\n", __func__, @@ -326,7 +331,7 @@ struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, struct nsm_handle *nsm = NULL; struct sockaddr *src_sap = svc_daddr(rqstp); size_t src_len = rqstp->rq_daddrlen; - struct net *net = rqstp->rq_xprt->xpt_net; + struct net *net = SVC_NET(rqstp); struct nlm_lookup_host_info ni = { .server = 1, .sap = svc_addr(rqstp), @@ -337,6 +342,7 @@ struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, .hostname_len = hostname_len, .net = net, }; + struct lockd_net *ln = net_generic(net, lockd_net_id); dprintk("lockd: %s(host='%*s', vers=%u, proto=%s)\n", __func__, (int)hostname_len, hostname, rqstp->rq_vers, @@ -344,8 +350,8 @@ struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, mutex_lock(&nlm_host_mutex); - if (time_after_eq(jiffies, next_gc)) - nlm_gc_hosts(); + if (time_after_eq(jiffies, ln->next_gc)) + nlm_gc_hosts(net); chain = &nlm_server_hosts[nlm_hash_address(ni.sap)]; hlist_for_each_entry(host, pos, chain, h_hash) { @@ -382,6 +388,7 @@ struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, memcpy(nlm_srcaddr(host), src_sap, src_len); host->h_srcaddrlen = src_len; hlist_add_head(&host->h_hash, chain); + ln->nrhosts++; nrhosts++; dprintk("lockd: %s created host %s (%s)\n", @@ -565,6 +572,35 @@ void nlm_host_rebooted(const struct nlm_reboot *info) nsm_release(nsm); } +static void nlm_complain_hosts(struct net *net) +{ + struct hlist_head *chain; + struct hlist_node *pos; + struct nlm_host *host; + + if (net) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + + if (ln->nrhosts == 0) + return; + printk(KERN_WARNING "lockd: couldn't shutdown host module for net %p!\n", net); + dprintk("lockd: %lu hosts left in net %p:\n", ln->nrhosts, net); + } else { + if (nrhosts == 0) + return; + printk(KERN_WARNING "lockd: couldn't shutdown host module!\n"); + dprintk("lockd: %lu hosts left:\n", nrhosts); + } + + for_each_host(host, pos, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; + dprintk(" %s (cnt %d use %d exp %ld net %p)\n", + host->h_name, atomic_read(&host->h_count), + host->h_inuse, host->h_expires, host->net); + } +} + void nlm_shutdown_hosts_net(struct net *net) { @@ -572,11 +608,10 @@ nlm_shutdown_hosts_net(struct net *net) struct hlist_node *pos; struct nlm_host *host; - dprintk("lockd: shutting down host module\n"); mutex_lock(&nlm_host_mutex); /* First, make all hosts eligible for gc */ - dprintk("lockd: nuking all hosts...\n"); + dprintk("lockd: nuking all hosts in net %p...\n", net); for_each_host(host, pos, chain, nlm_server_hosts) { if (net && host->net != net) continue; @@ -588,8 +623,10 @@ nlm_shutdown_hosts_net(struct net *net) } /* Then, perform a garbage collection pass */ - nlm_gc_hosts(); + nlm_gc_hosts(net); mutex_unlock(&nlm_host_mutex); + + nlm_complain_hosts(net); } /* @@ -599,22 +636,8 @@ nlm_shutdown_hosts_net(struct net *net) void nlm_shutdown_hosts(void) { - struct hlist_head *chain; - struct hlist_node *pos; - struct nlm_host *host; - + dprintk("lockd: shutting down host module\n"); nlm_shutdown_hosts_net(NULL); - - /* complain if any hosts are left */ - if (nrhosts != 0) { - printk(KERN_WARNING "lockd: couldn't shutdown host module!\n"); - dprintk("lockd: %lu hosts left:\n", nrhosts); - for_each_host(host, pos, chain, nlm_server_hosts) { - dprintk(" %s (cnt %d use %d exp %ld net %p)\n", - host->h_name, atomic_read(&host->h_count), - host->h_inuse, host->h_expires, host->net); - } - } } /* @@ -623,30 +646,39 @@ nlm_shutdown_hosts(void) * mark & sweep for resources held by remote clients. */ static void -nlm_gc_hosts(void) +nlm_gc_hosts(struct net *net) { struct hlist_head *chain; struct hlist_node *pos, *next; struct nlm_host *host; - dprintk("lockd: host garbage collection\n"); - for_each_host(host, pos, chain, nlm_server_hosts) + dprintk("lockd: host garbage collection for net %p\n", net); + for_each_host(host, pos, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; host->h_inuse = 0; + } /* Mark all hosts that hold locks, blocks or shares */ - nlmsvc_mark_resources(); + nlmsvc_mark_resources(net); for_each_host_safe(host, pos, next, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; if (atomic_read(&host->h_count) || host->h_inuse || time_before(jiffies, host->h_expires)) { dprintk("nlm_gc_hosts skipping %s " - "(cnt %d use %d exp %ld)\n", + "(cnt %d use %d exp %ld net %p)\n", host->h_name, atomic_read(&host->h_count), - host->h_inuse, host->h_expires); + host->h_inuse, host->h_expires, host->net); continue; } nlm_destroy_host_locked(host); } - next_gc = jiffies + NLM_HOST_COLLECT; + if (net) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + + ln->next_gc = jiffies + NLM_HOST_COLLECT; + } } diff --git a/fs/lockd/netns.h b/fs/lockd/netns.h index ce227e0fbc5..4eee248ba96 100644 --- a/fs/lockd/netns.h +++ b/fs/lockd/netns.h @@ -1,10 +1,17 @@ #ifndef __LOCKD_NETNS_H__ #define __LOCKD_NETNS_H__ +#include <linux/fs.h> #include <net/netns/generic.h> struct lockd_net { unsigned int nlmsvc_users; + unsigned long next_gc; + unsigned long nrhosts; + + struct delayed_work grace_period_end; + struct lock_manager lockd_manager; + struct list_head grace_list; }; extern int lockd_net_id; diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c index 80938fda67e..31a63f87b80 100644 --- a/fs/lockd/svc.c +++ b/fs/lockd/svc.c @@ -87,32 +87,36 @@ static unsigned long get_lockd_grace_period(void) return nlm_timeout * 5 * HZ; } -static struct lock_manager lockd_manager = { -}; - -static void grace_ender(struct work_struct *not_used) +static void grace_ender(struct work_struct *grace) { - locks_end_grace(&lockd_manager); -} + struct delayed_work *dwork = container_of(grace, struct delayed_work, + work); + struct lockd_net *ln = container_of(dwork, struct lockd_net, + grace_period_end); -static DECLARE_DELAYED_WORK(grace_period_end, grace_ender); + locks_end_grace(&ln->lockd_manager); +} -static void set_grace_period(void) +static void set_grace_period(struct net *net) { unsigned long grace_period = get_lockd_grace_period(); + struct lockd_net *ln = net_generic(net, lockd_net_id); - locks_start_grace(&lockd_manager); - cancel_delayed_work_sync(&grace_period_end); - schedule_delayed_work(&grace_period_end, grace_period); + locks_start_grace(net, &ln->lockd_manager); + cancel_delayed_work_sync(&ln->grace_period_end); + schedule_delayed_work(&ln->grace_period_end, grace_period); } static void restart_grace(void) { if (nlmsvc_ops) { - cancel_delayed_work_sync(&grace_period_end); - locks_end_grace(&lockd_manager); + struct net *net = &init_net; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + cancel_delayed_work_sync(&ln->grace_period_end); + locks_end_grace(&ln->lockd_manager); nlmsvc_invalidate_all(); - set_grace_period(); + set_grace_period(net); } } @@ -137,8 +141,6 @@ lockd(void *vrqstp) nlm_timeout = LOCKD_DFLT_TIMEO; nlmsvc_timeout = nlm_timeout * HZ; - set_grace_period(); - /* * The main request loop. We don't terminate until the last * NFS mount or NFS daemon has gone away. @@ -184,8 +186,6 @@ lockd(void *vrqstp) svc_process(rqstp); } flush_signals(current); - cancel_delayed_work_sync(&grace_period_end); - locks_end_grace(&lockd_manager); if (nlmsvc_ops) nlmsvc_invalidate_all(); nlm_shutdown_hosts(); @@ -266,6 +266,7 @@ static int lockd_up_net(struct svc_serv *serv, struct net *net) error = make_socks(serv, net); if (error < 0) goto err_socks; + set_grace_period(net); dprintk("lockd_up_net: per-net data created; net=%p\n", net); return 0; @@ -283,6 +284,8 @@ static void lockd_down_net(struct svc_serv *serv, struct net *net) if (ln->nlmsvc_users) { if (--ln->nlmsvc_users == 0) { nlm_shutdown_hosts_net(net); + cancel_delayed_work_sync(&ln->grace_period_end); + locks_end_grace(&ln->lockd_manager); svc_shutdown_net(serv, net); dprintk("lockd_down_net: per-net data destroyed; net=%p\n", net); } @@ -589,6 +592,10 @@ module_param(nlm_max_connections, uint, 0644); static int lockd_init_net(struct net *net) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + + INIT_DELAYED_WORK(&ln->grace_period_end, grace_ender); + INIT_LIST_HEAD(&ln->grace_list); return 0; } diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index 9a41fdc1951..4a43d253c04 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -11,6 +11,7 @@ #include <linux/time.h> #include <linux/lockd/lockd.h> #include <linux/lockd/share.h> +#include <linux/sunrpc/svc_xprt.h> #define NLMDBG_FACILITY NLMDBG_CLIENT @@ -151,7 +152,7 @@ nlm4svc_proc_cancel(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(SVC_NET(rqstp))) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -161,7 +162,7 @@ nlm4svc_proc_cancel(struct svc_rqst *rqstp, struct nlm_args *argp, return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; /* Try to cancel request. */ - resp->status = nlmsvc_cancel_blocked(file, &argp->lock); + resp->status = nlmsvc_cancel_blocked(SVC_NET(rqstp), file, &argp->lock); dprintk("lockd: CANCEL status %d\n", ntohl(resp->status)); nlmsvc_release_host(host); @@ -184,7 +185,7 @@ nlm4svc_proc_unlock(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept new lock requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(SVC_NET(rqstp))) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -194,7 +195,7 @@ nlm4svc_proc_unlock(struct svc_rqst *rqstp, struct nlm_args *argp, return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; /* Now try to remove the lock */ - resp->status = nlmsvc_unlock(file, &argp->lock); + resp->status = nlmsvc_unlock(SVC_NET(rqstp), file, &argp->lock); dprintk("lockd: UNLOCK status %d\n", ntohl(resp->status)); nlmsvc_release_host(host); @@ -321,7 +322,7 @@ nlm4svc_proc_share(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept new lock requests during grace period */ - if (locks_in_grace() && !argp->reclaim) { + if (locks_in_grace(SVC_NET(rqstp)) && !argp->reclaim) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -354,7 +355,7 @@ nlm4svc_proc_unshare(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(SVC_NET(rqstp))) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index e46353f41a4..afe4488c33d 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -26,7 +26,7 @@ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/sunrpc/clnt.h> -#include <linux/sunrpc/svc.h> +#include <linux/sunrpc/svc_xprt.h> #include <linux/lockd/nlm.h> #include <linux/lockd/lockd.h> #include <linux/kthread.h> @@ -447,11 +447,11 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, goto out; } - if (locks_in_grace() && !reclaim) { + if (locks_in_grace(SVC_NET(rqstp)) && !reclaim) { ret = nlm_lck_denied_grace_period; goto out; } - if (reclaim && !locks_in_grace()) { + if (reclaim && !locks_in_grace(SVC_NET(rqstp))) { ret = nlm_lck_denied_grace_period; goto out; } @@ -559,7 +559,7 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, goto out; } - if (locks_in_grace()) { + if (locks_in_grace(SVC_NET(rqstp))) { ret = nlm_lck_denied_grace_period; goto out; } @@ -603,7 +603,7 @@ out: * must be removed. */ __be32 -nlmsvc_unlock(struct nlm_file *file, struct nlm_lock *lock) +nlmsvc_unlock(struct net *net, struct nlm_file *file, struct nlm_lock *lock) { int error; @@ -615,7 +615,7 @@ nlmsvc_unlock(struct nlm_file *file, struct nlm_lock *lock) (long long)lock->fl.fl_end); /* First, cancel any lock that might be there */ - nlmsvc_cancel_blocked(file, lock); + nlmsvc_cancel_blocked(net, file, lock); lock->fl.fl_type = F_UNLCK; error = vfs_lock_file(file->f_file, F_SETLK, &lock->fl, NULL); @@ -631,7 +631,7 @@ nlmsvc_unlock(struct nlm_file *file, struct nlm_lock *lock) * The calling procedure must check whether the file can be closed. */ __be32 -nlmsvc_cancel_blocked(struct nlm_file *file, struct nlm_lock *lock) +nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct nlm_lock *lock) { struct nlm_block *block; int status = 0; @@ -643,7 +643,7 @@ nlmsvc_cancel_blocked(struct nlm_file *file, struct nlm_lock *lock) (long long)lock->fl.fl_start, (long long)lock->fl.fl_end); - if (locks_in_grace()) + if (locks_in_grace(net)) return nlm_lck_denied_grace_period; mutex_lock(&file->f_mutex); diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index d27aab11f32..de8f2caa223 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -11,6 +11,7 @@ #include <linux/time.h> #include <linux/lockd/lockd.h> #include <linux/lockd/share.h> +#include <linux/sunrpc/svc_xprt.h> #define NLMDBG_FACILITY NLMDBG_CLIENT @@ -175,13 +176,14 @@ nlmsvc_proc_cancel(struct svc_rqst *rqstp, struct nlm_args *argp, { struct nlm_host *host; struct nlm_file *file; + struct net *net = SVC_NET(rqstp); dprintk("lockd: CANCEL called\n"); resp->cookie = argp->cookie; /* Don't accept requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(net)) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -191,7 +193,7 @@ nlmsvc_proc_cancel(struct svc_rqst *rqstp, struct nlm_args *argp, return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; /* Try to cancel request. */ - resp->status = cast_status(nlmsvc_cancel_blocked(file, &argp->lock)); + resp->status = cast_status(nlmsvc_cancel_blocked(net, file, &argp->lock)); dprintk("lockd: CANCEL status %d\n", ntohl(resp->status)); nlmsvc_release_host(host); @@ -208,13 +210,14 @@ nlmsvc_proc_unlock(struct svc_rqst *rqstp, struct nlm_args *argp, { struct nlm_host *host; struct nlm_file *file; + struct net *net = SVC_NET(rqstp); dprintk("lockd: UNLOCK called\n"); resp->cookie = argp->cookie; /* Don't accept new lock requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(net)) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -224,7 +227,7 @@ nlmsvc_proc_unlock(struct svc_rqst *rqstp, struct nlm_args *argp, return resp->status == nlm_drop_reply ? rpc_drop_reply :rpc_success; /* Now try to remove the lock */ - resp->status = cast_status(nlmsvc_unlock(file, &argp->lock)); + resp->status = cast_status(nlmsvc_unlock(net, file, &argp->lock)); dprintk("lockd: UNLOCK status %d\n", ntohl(resp->status)); nlmsvc_release_host(host); @@ -361,7 +364,7 @@ nlmsvc_proc_share(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept new lock requests during grace period */ - if (locks_in_grace() && !argp->reclaim) { + if (locks_in_grace(SVC_NET(rqstp)) && !argp->reclaim) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } @@ -394,7 +397,7 @@ nlmsvc_proc_unshare(struct svc_rqst *rqstp, struct nlm_args *argp, resp->cookie = argp->cookie; /* Don't accept requests during grace period */ - if (locks_in_grace()) { + if (locks_in_grace(SVC_NET(rqstp))) { resp->status = nlm_lck_denied_grace_period; return rpc_success; } diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index 2240d384d78..0deb5f6c9dd 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -309,7 +309,8 @@ nlm_release_file(struct nlm_file *file) * Helpers function for resource traversal * * nlmsvc_mark_host: - * used by the garbage collector; simply sets h_inuse. + * used by the garbage collector; simply sets h_inuse only for those + * hosts, which passed network check. * Always returns 0. * * nlmsvc_same_host: @@ -320,12 +321,15 @@ nlm_release_file(struct nlm_file *file) * returns 1 iff the host is a client. * Used by nlmsvc_invalidate_all */ + static int -nlmsvc_mark_host(void *data, struct nlm_host *dummy) +nlmsvc_mark_host(void *data, struct nlm_host *hint) { struct nlm_host *host = data; - host->h_inuse = 1; + if ((hint->net == NULL) || + (host->net == hint->net)) + host->h_inuse = 1; return 0; } @@ -358,10 +362,13 @@ nlmsvc_is_client(void *data, struct nlm_host *dummy) * Mark all hosts that still hold resources */ void -nlmsvc_mark_resources(void) +nlmsvc_mark_resources(struct net *net) { - dprintk("lockd: nlmsvc_mark_resources\n"); - nlm_traverse_files(NULL, nlmsvc_mark_host, NULL); + struct nlm_host hint; + + dprintk("lockd: nlmsvc_mark_resources for net %p\n", net); + hint.net = net; + nlm_traverse_files(&hint, nlmsvc_mark_host, NULL); } /* |