Commit 9bbc052d authored by David S. Miller's avatar David S. Miller

Merge branch 'pablo/nf-2.6-updates' of git://1984.lsi.us.es/net-2.6

parents 0d4420a9 93bbce1a
...@@ -791,6 +791,7 @@ struct ip_vs_app { ...@@ -791,6 +791,7 @@ struct ip_vs_app {
/* IPVS in network namespace */ /* IPVS in network namespace */
struct netns_ipvs { struct netns_ipvs {
int gen; /* Generation */ int gen; /* Generation */
int enable; /* enable like nf_hooks do */
/* /*
* Hash table: for real service lookups * Hash table: for real service lookups
*/ */
...@@ -1089,6 +1090,22 @@ ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) ...@@ -1089,6 +1090,22 @@ ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp)
atomic_inc(&ctl_cp->n_control); atomic_inc(&ctl_cp->n_control);
} }
/*
* IPVS netns init & cleanup functions
*/
extern int __ip_vs_estimator_init(struct net *net);
extern int __ip_vs_control_init(struct net *net);
extern int __ip_vs_protocol_init(struct net *net);
extern int __ip_vs_app_init(struct net *net);
extern int __ip_vs_conn_init(struct net *net);
extern int __ip_vs_sync_init(struct net *net);
extern void __ip_vs_conn_cleanup(struct net *net);
extern void __ip_vs_app_cleanup(struct net *net);
extern void __ip_vs_protocol_cleanup(struct net *net);
extern void __ip_vs_control_cleanup(struct net *net);
extern void __ip_vs_estimator_cleanup(struct net *net);
extern void __ip_vs_sync_cleanup(struct net *net);
extern void __ip_vs_service_cleanup(struct net *net);
/* /*
* IPVS application functions * IPVS application functions
......
...@@ -1766,7 +1766,7 @@ static int compat_table_info(const struct ebt_table_info *info, ...@@ -1766,7 +1766,7 @@ static int compat_table_info(const struct ebt_table_info *info,
newinfo->entries_size = size; newinfo->entries_size = size;
xt_compat_init_offsets(AF_INET, info->nentries); xt_compat_init_offsets(NFPROTO_BRIDGE, info->nentries);
return EBT_ENTRY_ITERATE(entries, size, compat_calc_entry, info, return EBT_ENTRY_ITERATE(entries, size, compat_calc_entry, info,
entries, newinfo); entries, newinfo);
} }
...@@ -1882,7 +1882,7 @@ static int compat_mtw_from_user(struct compat_ebt_entry_mwt *mwt, ...@@ -1882,7 +1882,7 @@ static int compat_mtw_from_user(struct compat_ebt_entry_mwt *mwt,
struct xt_match *match; struct xt_match *match;
struct xt_target *wt; struct xt_target *wt;
void *dst = NULL; void *dst = NULL;
int off, pad = 0, ret = 0; int off, pad = 0;
unsigned int size_kern, entry_offset, match_size = mwt->match_size; unsigned int size_kern, entry_offset, match_size = mwt->match_size;
strlcpy(name, mwt->u.name, sizeof(name)); strlcpy(name, mwt->u.name, sizeof(name));
...@@ -1935,13 +1935,6 @@ static int compat_mtw_from_user(struct compat_ebt_entry_mwt *mwt, ...@@ -1935,13 +1935,6 @@ static int compat_mtw_from_user(struct compat_ebt_entry_mwt *mwt,
break; break;
} }
if (!dst) {
ret = xt_compat_add_offset(NFPROTO_BRIDGE, entry_offset,
off + ebt_compat_entry_padsize());
if (ret < 0)
return ret;
}
state->buf_kern_offset += match_size + off; state->buf_kern_offset += match_size + off;
state->buf_user_offset += match_size; state->buf_user_offset += match_size;
pad = XT_ALIGN(size_kern) - size_kern; pad = XT_ALIGN(size_kern) - size_kern;
...@@ -2016,50 +2009,6 @@ static int ebt_size_mwt(struct compat_ebt_entry_mwt *match32, ...@@ -2016,50 +2009,6 @@ static int ebt_size_mwt(struct compat_ebt_entry_mwt *match32,
return growth; return growth;
} }
#define EBT_COMPAT_WATCHER_ITERATE(e, fn, args...) \
({ \
unsigned int __i; \
int __ret = 0; \
struct compat_ebt_entry_mwt *__watcher; \
\
for (__i = e->watchers_offset; \
__i < (e)->target_offset; \
__i += __watcher->watcher_size + \
sizeof(struct compat_ebt_entry_mwt)) { \
__watcher = (void *)(e) + __i; \
__ret = fn(__watcher , ## args); \
if (__ret != 0) \
break; \
} \
if (__ret == 0) { \
if (__i != (e)->target_offset) \
__ret = -EINVAL; \
} \
__ret; \
})
#define EBT_COMPAT_MATCH_ITERATE(e, fn, args...) \
({ \
unsigned int __i; \
int __ret = 0; \
struct compat_ebt_entry_mwt *__match; \
\
for (__i = sizeof(struct ebt_entry); \
__i < (e)->watchers_offset; \
__i += __match->match_size + \
sizeof(struct compat_ebt_entry_mwt)) { \
__match = (void *)(e) + __i; \
__ret = fn(__match , ## args); \
if (__ret != 0) \
break; \
} \
if (__ret == 0) { \
if (__i != (e)->watchers_offset) \
__ret = -EINVAL; \
} \
__ret; \
})
/* called for all ebt_entry structures. */ /* called for all ebt_entry structures. */
static int size_entry_mwt(struct ebt_entry *entry, const unsigned char *base, static int size_entry_mwt(struct ebt_entry *entry, const unsigned char *base,
unsigned int *total, unsigned int *total,
...@@ -2132,6 +2081,14 @@ static int size_entry_mwt(struct ebt_entry *entry, const unsigned char *base, ...@@ -2132,6 +2081,14 @@ static int size_entry_mwt(struct ebt_entry *entry, const unsigned char *base,
} }
} }
if (state->buf_kern_start == NULL) {
unsigned int offset = buf_start - (char *) base;
ret = xt_compat_add_offset(NFPROTO_BRIDGE, offset, new_offset);
if (ret < 0)
return ret;
}
startoff = state->buf_user_offset - startoff; startoff = state->buf_user_offset - startoff;
BUG_ON(*total < startoff); BUG_ON(*total < startoff);
...@@ -2240,6 +2197,7 @@ static int compat_do_replace(struct net *net, void __user *user, ...@@ -2240,6 +2197,7 @@ static int compat_do_replace(struct net *net, void __user *user,
xt_compat_lock(NFPROTO_BRIDGE); xt_compat_lock(NFPROTO_BRIDGE);
xt_compat_init_offsets(NFPROTO_BRIDGE, tmp.nentries);
ret = compat_copy_entries(entries_tmp, tmp.entries_size, &state); ret = compat_copy_entries(entries_tmp, tmp.entries_size, &state);
if (ret < 0) if (ret < 0)
goto out_unlock; goto out_unlock;
......
...@@ -45,6 +45,8 @@ static void send_reset(struct net *net, struct sk_buff *oldskb) ...@@ -45,6 +45,8 @@ static void send_reset(struct net *net, struct sk_buff *oldskb)
int tcphoff, needs_ack; int tcphoff, needs_ack;
const struct ipv6hdr *oip6h = ipv6_hdr(oldskb); const struct ipv6hdr *oip6h = ipv6_hdr(oldskb);
struct ipv6hdr *ip6h; struct ipv6hdr *ip6h;
#define DEFAULT_TOS_VALUE 0x0U
const __u8 tclass = DEFAULT_TOS_VALUE;
struct dst_entry *dst = NULL; struct dst_entry *dst = NULL;
u8 proto; u8 proto;
struct flowi6 fl6; struct flowi6 fl6;
...@@ -124,7 +126,7 @@ static void send_reset(struct net *net, struct sk_buff *oldskb) ...@@ -124,7 +126,7 @@ static void send_reset(struct net *net, struct sk_buff *oldskb)
skb_put(nskb, sizeof(struct ipv6hdr)); skb_put(nskb, sizeof(struct ipv6hdr));
skb_reset_network_header(nskb); skb_reset_network_header(nskb);
ip6h = ipv6_hdr(nskb); ip6h = ipv6_hdr(nskb);
ip6h->version = 6; *(__be32 *)ip6h = htonl(0x60000000 | (tclass << 20));
ip6h->hop_limit = ip6_dst_hoplimit(dst); ip6h->hop_limit = ip6_dst_hoplimit(dst);
ip6h->nexthdr = IPPROTO_TCP; ip6h->nexthdr = IPPROTO_TCP;
ipv6_addr_copy(&ip6h->saddr, &oip6h->daddr); ipv6_addr_copy(&ip6h->saddr, &oip6h->daddr);
......
...@@ -576,7 +576,7 @@ static const struct file_operations ip_vs_app_fops = { ...@@ -576,7 +576,7 @@ static const struct file_operations ip_vs_app_fops = {
}; };
#endif #endif
static int __net_init __ip_vs_app_init(struct net *net) int __net_init __ip_vs_app_init(struct net *net)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
...@@ -585,26 +585,17 @@ static int __net_init __ip_vs_app_init(struct net *net) ...@@ -585,26 +585,17 @@ static int __net_init __ip_vs_app_init(struct net *net)
return 0; return 0;
} }
static void __net_exit __ip_vs_app_cleanup(struct net *net) void __net_exit __ip_vs_app_cleanup(struct net *net)
{ {
proc_net_remove(net, "ip_vs_app"); proc_net_remove(net, "ip_vs_app");
} }
static struct pernet_operations ip_vs_app_ops = {
.init = __ip_vs_app_init,
.exit = __ip_vs_app_cleanup,
};
int __init ip_vs_app_init(void) int __init ip_vs_app_init(void)
{ {
int rv; return 0;
rv = register_pernet_subsys(&ip_vs_app_ops);
return rv;
} }
void ip_vs_app_cleanup(void) void ip_vs_app_cleanup(void)
{ {
unregister_pernet_subsys(&ip_vs_app_ops);
} }
...@@ -1258,22 +1258,17 @@ int __net_init __ip_vs_conn_init(struct net *net) ...@@ -1258,22 +1258,17 @@ int __net_init __ip_vs_conn_init(struct net *net)
return 0; return 0;
} }
static void __net_exit __ip_vs_conn_cleanup(struct net *net) void __net_exit __ip_vs_conn_cleanup(struct net *net)
{ {
/* flush all the connection entries first */ /* flush all the connection entries first */
ip_vs_conn_flush(net); ip_vs_conn_flush(net);
proc_net_remove(net, "ip_vs_conn"); proc_net_remove(net, "ip_vs_conn");
proc_net_remove(net, "ip_vs_conn_sync"); proc_net_remove(net, "ip_vs_conn_sync");
} }
static struct pernet_operations ipvs_conn_ops = {
.init = __ip_vs_conn_init,
.exit = __ip_vs_conn_cleanup,
};
int __init ip_vs_conn_init(void) int __init ip_vs_conn_init(void)
{ {
int idx; int idx;
int retc;
/* Compute size and mask */ /* Compute size and mask */
ip_vs_conn_tab_size = 1 << ip_vs_conn_tab_bits; ip_vs_conn_tab_size = 1 << ip_vs_conn_tab_bits;
...@@ -1309,17 +1304,14 @@ int __init ip_vs_conn_init(void) ...@@ -1309,17 +1304,14 @@ int __init ip_vs_conn_init(void)
rwlock_init(&__ip_vs_conntbl_lock_array[idx].l); rwlock_init(&__ip_vs_conntbl_lock_array[idx].l);
} }
retc = register_pernet_subsys(&ipvs_conn_ops);
/* calculate the random value for connection hash */ /* calculate the random value for connection hash */
get_random_bytes(&ip_vs_conn_rnd, sizeof(ip_vs_conn_rnd)); get_random_bytes(&ip_vs_conn_rnd, sizeof(ip_vs_conn_rnd));
return retc; return 0;
} }
void ip_vs_conn_cleanup(void) void ip_vs_conn_cleanup(void)
{ {
unregister_pernet_subsys(&ipvs_conn_ops);
/* Release the empty cache */ /* Release the empty cache */
kmem_cache_destroy(ip_vs_conn_cachep); kmem_cache_destroy(ip_vs_conn_cachep);
vfree(ip_vs_conn_tab); vfree(ip_vs_conn_tab);
......
...@@ -1113,6 +1113,9 @@ ip_vs_out(unsigned int hooknum, struct sk_buff *skb, int af) ...@@ -1113,6 +1113,9 @@ ip_vs_out(unsigned int hooknum, struct sk_buff *skb, int af)
return NF_ACCEPT; return NF_ACCEPT;
net = skb_net(skb); net = skb_net(skb);
if (!net_ipvs(net)->enable)
return NF_ACCEPT;
ip_vs_fill_iphdr(af, skb_network_header(skb), &iph); ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
#ifdef CONFIG_IP_VS_IPV6 #ifdef CONFIG_IP_VS_IPV6
if (af == AF_INET6) { if (af == AF_INET6) {
...@@ -1343,6 +1346,7 @@ ip_vs_in_icmp(struct sk_buff *skb, int *related, unsigned int hooknum) ...@@ -1343,6 +1346,7 @@ ip_vs_in_icmp(struct sk_buff *skb, int *related, unsigned int hooknum)
return NF_ACCEPT; /* The packet looks wrong, ignore */ return NF_ACCEPT; /* The packet looks wrong, ignore */
net = skb_net(skb); net = skb_net(skb);
pd = ip_vs_proto_data_get(net, cih->protocol); pd = ip_vs_proto_data_get(net, cih->protocol);
if (!pd) if (!pd)
return NF_ACCEPT; return NF_ACCEPT;
...@@ -1529,6 +1533,11 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) ...@@ -1529,6 +1533,11 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)
IP_VS_DBG_ADDR(af, &iph.daddr), hooknum); IP_VS_DBG_ADDR(af, &iph.daddr), hooknum);
return NF_ACCEPT; return NF_ACCEPT;
} }
/* ipvs enabled in this netns ? */
net = skb_net(skb);
if (!net_ipvs(net)->enable)
return NF_ACCEPT;
ip_vs_fill_iphdr(af, skb_network_header(skb), &iph); ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
/* Bad... Do not break raw sockets */ /* Bad... Do not break raw sockets */
...@@ -1562,7 +1571,6 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) ...@@ -1562,7 +1571,6 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)
ip_vs_fill_iphdr(af, skb_network_header(skb), &iph); ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
} }
net = skb_net(skb);
/* Protocol supported? */ /* Protocol supported? */
pd = ip_vs_proto_data_get(net, iph.protocol); pd = ip_vs_proto_data_get(net, iph.protocol);
if (unlikely(!pd)) if (unlikely(!pd))
...@@ -1588,7 +1596,6 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) ...@@ -1588,7 +1596,6 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)
} }
IP_VS_DBG_PKT(11, af, pp, skb, 0, "Incoming packet"); IP_VS_DBG_PKT(11, af, pp, skb, 0, "Incoming packet");
net = skb_net(skb);
ipvs = net_ipvs(net); ipvs = net_ipvs(net);
/* Check the server status */ /* Check the server status */
if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) { if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
...@@ -1743,10 +1750,16 @@ ip_vs_forward_icmp(unsigned int hooknum, struct sk_buff *skb, ...@@ -1743,10 +1750,16 @@ ip_vs_forward_icmp(unsigned int hooknum, struct sk_buff *skb,
int (*okfn)(struct sk_buff *)) int (*okfn)(struct sk_buff *))
{ {
int r; int r;
struct net *net;
if (ip_hdr(skb)->protocol != IPPROTO_ICMP) if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
return NF_ACCEPT; return NF_ACCEPT;
/* ipvs enabled in this netns ? */
net = skb_net(skb);
if (!net_ipvs(net)->enable)
return NF_ACCEPT;
return ip_vs_in_icmp(skb, &r, hooknum); return ip_vs_in_icmp(skb, &r, hooknum);
} }
...@@ -1757,10 +1770,16 @@ ip_vs_forward_icmp_v6(unsigned int hooknum, struct sk_buff *skb, ...@@ -1757,10 +1770,16 @@ ip_vs_forward_icmp_v6(unsigned int hooknum, struct sk_buff *skb,
int (*okfn)(struct sk_buff *)) int (*okfn)(struct sk_buff *))
{ {
int r; int r;
struct net *net;
if (ipv6_hdr(skb)->nexthdr != IPPROTO_ICMPV6) if (ipv6_hdr(skb)->nexthdr != IPPROTO_ICMPV6)
return NF_ACCEPT; return NF_ACCEPT;
/* ipvs enabled in this netns ? */
net = skb_net(skb);
if (!net_ipvs(net)->enable)
return NF_ACCEPT;
return ip_vs_in_icmp_v6(skb, &r, hooknum); return ip_vs_in_icmp_v6(skb, &r, hooknum);
} }
#endif #endif
...@@ -1884,19 +1903,70 @@ static int __net_init __ip_vs_init(struct net *net) ...@@ -1884,19 +1903,70 @@ static int __net_init __ip_vs_init(struct net *net)
pr_err("%s(): no memory.\n", __func__); pr_err("%s(): no memory.\n", __func__);
return -ENOMEM; return -ENOMEM;
} }
/* Hold the beast until a service is registerd */
ipvs->enable = 0;
ipvs->net = net; ipvs->net = net;
/* Counters used for creating unique names */ /* Counters used for creating unique names */
ipvs->gen = atomic_read(&ipvs_netns_cnt); ipvs->gen = atomic_read(&ipvs_netns_cnt);
atomic_inc(&ipvs_netns_cnt); atomic_inc(&ipvs_netns_cnt);
net->ipvs = ipvs; net->ipvs = ipvs;
if (__ip_vs_estimator_init(net) < 0)
goto estimator_fail;
if (__ip_vs_control_init(net) < 0)
goto control_fail;
if (__ip_vs_protocol_init(net) < 0)
goto protocol_fail;
if (__ip_vs_app_init(net) < 0)
goto app_fail;
if (__ip_vs_conn_init(net) < 0)
goto conn_fail;
if (__ip_vs_sync_init(net) < 0)
goto sync_fail;
printk(KERN_INFO "IPVS: Creating netns size=%zu id=%d\n", printk(KERN_INFO "IPVS: Creating netns size=%zu id=%d\n",
sizeof(struct netns_ipvs), ipvs->gen); sizeof(struct netns_ipvs), ipvs->gen);
return 0; return 0;
/*
* Error handling
*/
sync_fail:
__ip_vs_conn_cleanup(net);
conn_fail:
__ip_vs_app_cleanup(net);
app_fail:
__ip_vs_protocol_cleanup(net);
protocol_fail:
__ip_vs_control_cleanup(net);
control_fail:
__ip_vs_estimator_cleanup(net);
estimator_fail:
return -ENOMEM;
} }
static void __net_exit __ip_vs_cleanup(struct net *net) static void __net_exit __ip_vs_cleanup(struct net *net)
{ {
IP_VS_DBG(10, "ipvs netns %d released\n", net_ipvs(net)->gen); __ip_vs_service_cleanup(net); /* ip_vs_flush() with locks */
__ip_vs_conn_cleanup(net);
__ip_vs_app_cleanup(net);
__ip_vs_protocol_cleanup(net);
__ip_vs_control_cleanup(net);
__ip_vs_estimator_cleanup(net);
IP_VS_DBG(2, "ipvs netns %d released\n", net_ipvs(net)->gen);
}
static void __net_exit __ip_vs_dev_cleanup(struct net *net)
{
EnterFunction(2);
net_ipvs(net)->enable = 0; /* Disable packet reception */
__ip_vs_sync_cleanup(net);
LeaveFunction(2);
} }
static struct pernet_operations ipvs_core_ops = { static struct pernet_operations ipvs_core_ops = {
...@@ -1906,6 +1976,10 @@ static struct pernet_operations ipvs_core_ops = { ...@@ -1906,6 +1976,10 @@ static struct pernet_operations ipvs_core_ops = {
.size = sizeof(struct netns_ipvs), .size = sizeof(struct netns_ipvs),
}; };
static struct pernet_operations ipvs_core_dev_ops = {
.exit = __ip_vs_dev_cleanup,
};
/* /*
* Initialize IP Virtual Server * Initialize IP Virtual Server
*/ */
...@@ -1913,10 +1987,6 @@ static int __init ip_vs_init(void) ...@@ -1913,10 +1987,6 @@ static int __init ip_vs_init(void)
{ {
int ret; int ret;
ret = register_pernet_subsys(&ipvs_core_ops); /* Alloc ip_vs struct */
if (ret < 0)
return ret;
ip_vs_estimator_init(); ip_vs_estimator_init();
ret = ip_vs_control_init(); ret = ip_vs_control_init();
if (ret < 0) { if (ret < 0) {
...@@ -1944,15 +2014,28 @@ static int __init ip_vs_init(void) ...@@ -1944,15 +2014,28 @@ static int __init ip_vs_init(void)
goto cleanup_conn; goto cleanup_conn;
} }
ret = register_pernet_subsys(&ipvs_core_ops); /* Alloc ip_vs struct */
if (ret < 0)
goto cleanup_sync;
ret = register_pernet_device(&ipvs_core_dev_ops);
if (ret < 0)
goto cleanup_sub;
ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops)); ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
if (ret < 0) { if (ret < 0) {
pr_err("can't register hooks.\n"); pr_err("can't register hooks.\n");
goto cleanup_sync; goto cleanup_dev;
} }
pr_info("ipvs loaded.\n"); pr_info("ipvs loaded.\n");
return ret; return ret;
cleanup_dev:
unregister_pernet_device(&ipvs_core_dev_ops);
cleanup_sub:
unregister_pernet_subsys(&ipvs_core_ops);
cleanup_sync: cleanup_sync:
ip_vs_sync_cleanup(); ip_vs_sync_cleanup();
cleanup_conn: cleanup_conn:
...@@ -1964,20 +2047,20 @@ static int __init ip_vs_init(void) ...@@ -1964,20 +2047,20 @@ static int __init ip_vs_init(void)
ip_vs_control_cleanup(); ip_vs_control_cleanup();
cleanup_estimator: cleanup_estimator:
ip_vs_estimator_cleanup(); ip_vs_estimator_cleanup();
unregister_pernet_subsys(&ipvs_core_ops); /* free ip_vs struct */
return ret; return ret;
} }
static void __exit ip_vs_cleanup(void) static void __exit ip_vs_cleanup(void)
{ {
nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops)); nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
unregister_pernet_device(&ipvs_core_dev_ops);
unregister_pernet_subsys(&ipvs_core_ops); /* free ip_vs struct */
ip_vs_sync_cleanup(); ip_vs_sync_cleanup();
ip_vs_conn_cleanup(); ip_vs_conn_cleanup();
ip_vs_app_cleanup(); ip_vs_app_cleanup();
ip_vs_protocol_cleanup(); ip_vs_protocol_cleanup();
ip_vs_control_cleanup(); ip_vs_control_cleanup();
ip_vs_estimator_cleanup(); ip_vs_estimator_cleanup();
unregister_pernet_subsys(&ipvs_core_ops); /* free ip_vs struct */
pr_info("ipvs unloaded.\n"); pr_info("ipvs unloaded.\n");
} }
......
...@@ -69,6 +69,11 @@ int ip_vs_get_debug_level(void) ...@@ -69,6 +69,11 @@ int ip_vs_get_debug_level(void)
} }
#endif #endif
/* Protos */
static void __ip_vs_del_service(struct ip_vs_service *svc);
#ifdef CONFIG_IP_VS_IPV6 #ifdef CONFIG_IP_VS_IPV6
/* Taken from rt6_fill_node() in net/ipv6/route.c, is there a better way? */ /* Taken from rt6_fill_node() in net/ipv6/route.c, is there a better way? */
static int __ip_vs_addr_is_local_v6(struct net *net, static int __ip_vs_addr_is_local_v6(struct net *net,
...@@ -1214,6 +1219,8 @@ ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u, ...@@ -1214,6 +1219,8 @@ ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u,
write_unlock_bh(&__ip_vs_svc_lock); write_unlock_bh(&__ip_vs_svc_lock);
*svc_p = svc; *svc_p = svc;
/* Now there is a service - full throttle */
ipvs->enable = 1;
return 0; return 0;
...@@ -1472,6 +1479,84 @@ static int ip_vs_flush(struct net *net) ...@@ -1472,6 +1479,84 @@ static int ip_vs_flush(struct net *net)
return 0; return 0;
} }
/*
* Delete service by {netns} in the service table.
* Called by __ip_vs_cleanup()
*/
void __ip_vs_service_cleanup(struct net *net)
{
EnterFunction(2);
/* Check for "full" addressed entries */
mutex_lock(&__ip_vs_mutex);
ip_vs_flush(net);
mutex_unlock(&__ip_vs_mutex);
LeaveFunction(2);
}
/*
* Release dst hold by dst_cache
*/
static inline void
__ip_vs_dev_reset(struct ip_vs_dest *dest, struct net_device *dev)
{
spin_lock_bh(&dest->dst_lock);
if (dest->dst_cache && dest->dst_cache->dev == dev) {
IP_VS_DBG_BUF(3, "Reset dev:%s dest %s:%u ,dest->refcnt=%d\n",
dev->name,
IP_VS_DBG_ADDR(dest->af, &dest->addr),
ntohs(dest->port),
atomic_read(&dest->refcnt));
ip_vs_dst_reset(dest);
}
spin_unlock_bh(&dest->dst_lock);
}
/*
* Netdev event receiver
* Currently only NETDEV_UNREGISTER is handled, i.e. if we hold a reference to
* a device that is "unregister" it must be released.
*/
static int ip_vs_dst_event(struct notifier_block *this, unsigned long event,
void *ptr)
{
struct net_device *dev = ptr;
struct net *net = dev_net(dev);
struct ip_vs_service *svc;
struct ip_vs_dest *dest;
unsigned int idx;
if (event != NETDEV_UNREGISTER)
return NOTIFY_DONE;
IP_VS_DBG(3, "%s() dev=%s\n", __func__, dev->name);
EnterFunction(2);
mutex_lock(&__ip_vs_mutex);
for (idx = 0; idx < IP_VS_SVC_TAB_SIZE; idx++) {
list_for_each_entry(svc, &ip_vs_svc_table[idx], s_list) {
if (net_eq(svc->net, net)) {
list_for_each_entry(dest, &svc->destinations,
n_list) {
__ip_vs_dev_reset(dest, dev);
}
}
}
list_for_each_entry(svc, &ip_vs_svc_fwm_table[idx], f_list) {
if (net_eq(svc->net, net)) {
list_for_each_entry(dest, &svc->destinations,
n_list) {
__ip_vs_dev_reset(dest, dev);
}
}
}
}
list_for_each_entry(dest, &net_ipvs(net)->dest_trash, n_list) {
__ip_vs_dev_reset(dest, dev);
}
mutex_unlock(&__ip_vs_mutex);
LeaveFunction(2);
return NOTIFY_DONE;
}
/* /*
* Zero counters in a service or all services * Zero counters in a service or all services
...@@ -3588,6 +3673,10 @@ void __net_init __ip_vs_control_cleanup_sysctl(struct net *net) { } ...@@ -3588,6 +3673,10 @@ void __net_init __ip_vs_control_cleanup_sysctl(struct net *net) { }
#endif #endif
static struct notifier_block ip_vs_dst_notifier = {
.notifier_call = ip_vs_dst_event,
};
int __net_init __ip_vs_control_init(struct net *net) int __net_init __ip_vs_control_init(struct net *net)
{ {
int idx; int idx;
...@@ -3626,7 +3715,7 @@ int __net_init __ip_vs_control_init(struct net *net) ...@@ -3626,7 +3715,7 @@ int __net_init __ip_vs_control_init(struct net *net)
return -ENOMEM; return -ENOMEM;
} }
static void __net_exit __ip_vs_control_cleanup(struct net *net) void __net_exit __ip_vs_control_cleanup(struct net *net)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
...@@ -3639,11 +3728,6 @@ static void __net_exit __ip_vs_control_cleanup(struct net *net) ...@@ -3639,11 +3728,6 @@ static void __net_exit __ip_vs_control_cleanup(struct net *net)
free_percpu(ipvs->tot_stats.cpustats); free_percpu(ipvs->tot_stats.cpustats);
} }
static struct pernet_operations ipvs_control_ops = {
.init = __ip_vs_control_init,
.exit = __ip_vs_control_cleanup,
};
int __init ip_vs_control_init(void) int __init ip_vs_control_init(void)
{ {
int idx; int idx;
...@@ -3657,33 +3741,32 @@ int __init ip_vs_control_init(void) ...@@ -3657,33 +3741,32 @@ int __init ip_vs_control_init(void)
INIT_LIST_HEAD(&ip_vs_svc_fwm_table[idx]); INIT_LIST_HEAD(&ip_vs_svc_fwm_table[idx]);
} }
ret = register_pernet_subsys(&ipvs_control_ops);
if (ret) {
pr_err("cannot register namespace.\n");
goto err;
}
smp_wmb(); /* Do we really need it now ? */ smp_wmb(); /* Do we really need it now ? */
ret = nf_register_sockopt(&ip_vs_sockopts); ret = nf_register_sockopt(&ip_vs_sockopts);
if (ret) { if (ret) {
pr_err("cannot register sockopt.\n"); pr_err("cannot register sockopt.\n");
goto err_net; goto err_sock;
} }
ret = ip_vs_genl_register(); ret = ip_vs_genl_register();
if (ret) { if (ret) {
pr_err("cannot register Generic Netlink interface.\n"); pr_err("cannot register Generic Netlink interface.\n");
nf_unregister_sockopt(&ip_vs_sockopts); goto err_genl;
goto err_net;
} }
ret = register_netdevice_notifier(&ip_vs_dst_notifier);
if (ret < 0)
goto err_notf;
LeaveFunction(2); LeaveFunction(2);
return 0; return 0;
err_net: err_notf:
unregister_pernet_subsys(&ipvs_control_ops); ip_vs_genl_unregister();
err: err_genl:
nf_unregister_sockopt(&ip_vs_sockopts);
err_sock:
return ret; return ret;
} }
...@@ -3691,7 +3774,6 @@ int __init ip_vs_control_init(void) ...@@ -3691,7 +3774,6 @@ int __init ip_vs_control_init(void)
void ip_vs_control_cleanup(void) void ip_vs_control_cleanup(void)
{ {
EnterFunction(2); EnterFunction(2);
unregister_pernet_subsys(&ipvs_control_ops);
ip_vs_genl_unregister(); ip_vs_genl_unregister();
nf_unregister_sockopt(&ip_vs_sockopts); nf_unregister_sockopt(&ip_vs_sockopts);
LeaveFunction(2); LeaveFunction(2);
......
...@@ -192,7 +192,7 @@ void ip_vs_read_estimator(struct ip_vs_stats_user *dst, ...@@ -192,7 +192,7 @@ void ip_vs_read_estimator(struct ip_vs_stats_user *dst,
dst->outbps = (e->outbps + 0xF) >> 5; dst->outbps = (e->outbps + 0xF) >> 5;
} }
static int __net_init __ip_vs_estimator_init(struct net *net) int __net_init __ip_vs_estimator_init(struct net *net)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
...@@ -203,24 +203,16 @@ static int __net_init __ip_vs_estimator_init(struct net *net) ...@@ -203,24 +203,16 @@ static int __net_init __ip_vs_estimator_init(struct net *net)
return 0; return 0;
} }
static void __net_exit __ip_vs_estimator_exit(struct net *net) void __net_exit __ip_vs_estimator_cleanup(struct net *net)
{ {
del_timer_sync(&net_ipvs(net)->est_timer); del_timer_sync(&net_ipvs(net)->est_timer);
} }
static struct pernet_operations ip_vs_app_ops = {
.init = __ip_vs_estimator_init,
.exit = __ip_vs_estimator_exit,
};
int __init ip_vs_estimator_init(void) int __init ip_vs_estimator_init(void)
{ {
int rv; return 0;
rv = register_pernet_subsys(&ip_vs_app_ops);
return rv;
} }
void ip_vs_estimator_cleanup(void) void ip_vs_estimator_cleanup(void)
{ {
unregister_pernet_subsys(&ip_vs_app_ops);
} }
...@@ -316,7 +316,7 @@ ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, ...@@ -316,7 +316,7 @@ ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp,
/* /*
* per network name-space init * per network name-space init
*/ */
static int __net_init __ip_vs_protocol_init(struct net *net) int __net_init __ip_vs_protocol_init(struct net *net)
{ {
#ifdef CONFIG_IP_VS_PROTO_TCP #ifdef CONFIG_IP_VS_PROTO_TCP
register_ip_vs_proto_netns(net, &ip_vs_protocol_tcp); register_ip_vs_proto_netns(net, &ip_vs_protocol_tcp);
...@@ -336,7 +336,7 @@ static int __net_init __ip_vs_protocol_init(struct net *net) ...@@ -336,7 +336,7 @@ static int __net_init __ip_vs_protocol_init(struct net *net)
return 0; return 0;
} }
static void __net_exit __ip_vs_protocol_cleanup(struct net *net) void __net_exit __ip_vs_protocol_cleanup(struct net *net)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
struct ip_vs_proto_data *pd; struct ip_vs_proto_data *pd;
...@@ -349,11 +349,6 @@ static void __net_exit __ip_vs_protocol_cleanup(struct net *net) ...@@ -349,11 +349,6 @@ static void __net_exit __ip_vs_protocol_cleanup(struct net *net)
} }
} }
static struct pernet_operations ipvs_proto_ops = {
.init = __ip_vs_protocol_init,
.exit = __ip_vs_protocol_cleanup,
};
int __init ip_vs_protocol_init(void) int __init ip_vs_protocol_init(void)
{ {
char protocols[64]; char protocols[64];
...@@ -382,7 +377,6 @@ int __init ip_vs_protocol_init(void) ...@@ -382,7 +377,6 @@ int __init ip_vs_protocol_init(void)
REGISTER_PROTOCOL(&ip_vs_protocol_esp); REGISTER_PROTOCOL(&ip_vs_protocol_esp);
#endif #endif
pr_info("Registered protocols (%s)\n", &protocols[2]); pr_info("Registered protocols (%s)\n", &protocols[2]);
return register_pernet_subsys(&ipvs_proto_ops);
return 0; return 0;
} }
...@@ -393,7 +387,6 @@ void ip_vs_protocol_cleanup(void) ...@@ -393,7 +387,6 @@ void ip_vs_protocol_cleanup(void)
struct ip_vs_protocol *pp; struct ip_vs_protocol *pp;
int i; int i;
unregister_pernet_subsys(&ipvs_proto_ops);
/* unregister all the ipvs protocols */ /* unregister all the ipvs protocols */
for (i = 0; i < IP_VS_PROTO_TAB_SIZE; i++) { for (i = 0; i < IP_VS_PROTO_TAB_SIZE; i++) {
while ((pp = ip_vs_proto_table[i]) != NULL) while ((pp = ip_vs_proto_table[i]) != NULL)
......
...@@ -1303,13 +1303,18 @@ static struct socket *make_send_sock(struct net *net) ...@@ -1303,13 +1303,18 @@ static struct socket *make_send_sock(struct net *net)
struct socket *sock; struct socket *sock;
int result; int result;
/* First create a socket */ /* First create a socket move it to right name space later */
result = __sock_create(net, PF_INET, SOCK_DGRAM, IPPROTO_UDP, &sock, 1); result = sock_create_kern(PF_INET, SOCK_DGRAM, IPPROTO_UDP, &sock);
if (result < 0) { if (result < 0) {
pr_err("Error during creation of socket; terminating\n"); pr_err("Error during creation of socket; terminating\n");
return ERR_PTR(result); return ERR_PTR(result);
} }
/*
* Kernel sockets that are a part of a namespace, should not
* hold a reference to a namespace in order to allow to stop it.
* After sk_change_net should be released using sk_release_kernel.
*/
sk_change_net(sock->sk, net);
result = set_mcast_if(sock->sk, ipvs->master_mcast_ifn); result = set_mcast_if(sock->sk, ipvs->master_mcast_ifn);
if (result < 0) { if (result < 0) {
pr_err("Error setting outbound mcast interface\n"); pr_err("Error setting outbound mcast interface\n");
...@@ -1334,8 +1339,8 @@ static struct socket *make_send_sock(struct net *net) ...@@ -1334,8 +1339,8 @@ static struct socket *make_send_sock(struct net *net)
return sock; return sock;
error: error:
sock_release(sock); sk_release_kernel(sock->sk);
return ERR_PTR(result); return ERR_PTR(result);
} }
...@@ -1350,12 +1355,17 @@ static struct socket *make_receive_sock(struct net *net) ...@@ -1350,12 +1355,17 @@ static struct socket *make_receive_sock(struct net *net)
int result; int result;
/* First create a socket */ /* First create a socket */
result = __sock_create(net, PF_INET, SOCK_DGRAM, IPPROTO_UDP, &sock, 1); result = sock_create_kern(PF_INET, SOCK_DGRAM, IPPROTO_UDP, &sock);
if (result < 0) { if (result < 0) {
pr_err("Error during creation of socket; terminating\n"); pr_err("Error during creation of socket; terminating\n");
return ERR_PTR(result); return ERR_PTR(result);
} }
/*
* Kernel sockets that are a part of a namespace, should not
* hold a reference to a namespace in order to allow to stop it.
* After sk_change_net should be released using sk_release_kernel.
*/
sk_change_net(sock->sk, net);
/* it is equivalent to the REUSEADDR option in user-space */ /* it is equivalent to the REUSEADDR option in user-space */
sock->sk->sk_reuse = 1; sock->sk->sk_reuse = 1;
...@@ -1377,8 +1387,8 @@ static struct socket *make_receive_sock(struct net *net) ...@@ -1377,8 +1387,8 @@ static struct socket *make_receive_sock(struct net *net)
return sock; return sock;
error: error:
sock_release(sock); sk_release_kernel(sock->sk);
return ERR_PTR(result); return ERR_PTR(result);
} }
...@@ -1473,7 +1483,7 @@ static int sync_thread_master(void *data) ...@@ -1473,7 +1483,7 @@ static int sync_thread_master(void *data)
ip_vs_sync_buff_release(sb); ip_vs_sync_buff_release(sb);
/* release the sending multicast socket */ /* release the sending multicast socket */
sock_release(tinfo->sock); sk_release_kernel(tinfo->sock->sk);
kfree(tinfo); kfree(tinfo);
return 0; return 0;
...@@ -1513,7 +1523,7 @@ static int sync_thread_backup(void *data) ...@@ -1513,7 +1523,7 @@ static int sync_thread_backup(void *data)
} }
/* release the sending multicast socket */ /* release the sending multicast socket */
sock_release(tinfo->sock); sk_release_kernel(tinfo->sock->sk);
kfree(tinfo->buf); kfree(tinfo->buf);
kfree(tinfo); kfree(tinfo);
...@@ -1601,7 +1611,7 @@ int start_sync_thread(struct net *net, int state, char *mcast_ifn, __u8 syncid) ...@@ -1601,7 +1611,7 @@ int start_sync_thread(struct net *net, int state, char *mcast_ifn, __u8 syncid)
outbuf: outbuf:
kfree(buf); kfree(buf);
outsocket: outsocket:
sock_release(sock); sk_release_kernel(sock->sk);
out: out:
return result; return result;
} }
...@@ -1610,6 +1620,7 @@ int start_sync_thread(struct net *net, int state, char *mcast_ifn, __u8 syncid) ...@@ -1610,6 +1620,7 @@ int start_sync_thread(struct net *net, int state, char *mcast_ifn, __u8 syncid)
int stop_sync_thread(struct net *net, int state) int stop_sync_thread(struct net *net, int state)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
int retc = -EINVAL;
IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current)); IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current));
...@@ -1629,7 +1640,7 @@ int stop_sync_thread(struct net *net, int state) ...@@ -1629,7 +1640,7 @@ int stop_sync_thread(struct net *net, int state)
spin_lock_bh(&ipvs->sync_lock); spin_lock_bh(&ipvs->sync_lock);
ipvs->sync_state &= ~IP_VS_STATE_MASTER; ipvs->sync_state &= ~IP_VS_STATE_MASTER;
spin_unlock_bh(&ipvs->sync_lock); spin_unlock_bh(&ipvs->sync_lock);
kthread_stop(ipvs->master_thread); retc = kthread_stop(ipvs->master_thread);
ipvs->master_thread = NULL; ipvs->master_thread = NULL;
} else if (state == IP_VS_STATE_BACKUP) { } else if (state == IP_VS_STATE_BACKUP) {
if (!ipvs->backup_thread) if (!ipvs->backup_thread)
...@@ -1639,22 +1650,20 @@ int stop_sync_thread(struct net *net, int state) ...@@ -1639,22 +1650,20 @@ int stop_sync_thread(struct net *net, int state)
task_pid_nr(ipvs->backup_thread)); task_pid_nr(ipvs->backup_thread));
ipvs->sync_state &= ~IP_VS_STATE_BACKUP; ipvs->sync_state &= ~IP_VS_STATE_BACKUP;
kthread_stop(ipvs->backup_thread); retc = kthread_stop(ipvs->backup_thread);
ipvs->backup_thread = NULL; ipvs->backup_thread = NULL;
} else {
return -EINVAL;
} }
/* decrease the module use count */ /* decrease the module use count */
ip_vs_use_count_dec(); ip_vs_use_count_dec();
return 0; return retc;
} }
/* /*
* Initialize data struct for each netns * Initialize data struct for each netns
*/ */
static int __net_init __ip_vs_sync_init(struct net *net) int __net_init __ip_vs_sync_init(struct net *net)
{ {
struct netns_ipvs *ipvs = net_ipvs(net); struct netns_ipvs *ipvs = net_ipvs(net);
...@@ -1668,24 +1677,24 @@ static int __net_init __ip_vs_sync_init(struct net *net) ...@@ -1668,24 +1677,24 @@ static int __net_init __ip_vs_sync_init(struct net *net)
return 0; return 0;
} }
static void __ip_vs_sync_cleanup(struct net *net) void __ip_vs_sync_cleanup(struct net *net)
{ {
stop_sync_thread(net, IP_VS_STATE_MASTER); int retc;
stop_sync_thread(net, IP_VS_STATE_BACKUP);
}
static struct pernet_operations ipvs_sync_ops = { retc = stop_sync_thread(net, IP_VS_STATE_MASTER);
.init = __ip_vs_sync_init, if (retc && retc != -ESRCH)
.exit = __ip_vs_sync_cleanup, pr_err("Failed to stop Master Daemon\n");
};
retc = stop_sync_thread(net, IP_VS_STATE_BACKUP);
if (retc && retc != -ESRCH)
pr_err("Failed to stop Backup Daemon\n");
}
int __init ip_vs_sync_init(void) int __init ip_vs_sync_init(void)
{ {
return register_pernet_subsys(&ipvs_sync_ops); return 0;
} }
void ip_vs_sync_cleanup(void) void ip_vs_sync_cleanup(void)
{ {
unregister_pernet_subsys(&ipvs_sync_ops);
} }
...@@ -1334,6 +1334,7 @@ ctnetlink_create_conntrack(struct net *net, u16 zone, ...@@ -1334,6 +1334,7 @@ ctnetlink_create_conntrack(struct net *net, u16 zone,
struct nf_conn *ct; struct nf_conn *ct;
int err = -EINVAL; int err = -EINVAL;
struct nf_conntrack_helper *helper; struct nf_conntrack_helper *helper;
struct nf_conn_tstamp *tstamp;
ct = nf_conntrack_alloc(net, zone, otuple, rtuple, GFP_ATOMIC); ct = nf_conntrack_alloc(net, zone, otuple, rtuple, GFP_ATOMIC);
if (IS_ERR(ct)) if (IS_ERR(ct))
...@@ -1451,6 +1452,9 @@ ctnetlink_create_conntrack(struct net *net, u16 zone, ...@@ -1451,6 +1452,9 @@ ctnetlink_create_conntrack(struct net *net, u16 zone,
__set_bit(IPS_EXPECTED_BIT, &ct->status); __set_bit(IPS_EXPECTED_BIT, &ct->status);
ct->master = master_ct; ct->master = master_ct;
} }
tstamp = nf_conn_tstamp_find(ct);
if (tstamp)
tstamp->start = ktime_to_ns(ktime_get_real());
add_timer(&ct->timeout); add_timer(&ct->timeout);
nf_conntrack_hash_insert(ct); nf_conntrack_hash_insert(ct);
......
...@@ -455,6 +455,7 @@ void xt_compat_flush_offsets(u_int8_t af) ...@@ -455,6 +455,7 @@ void xt_compat_flush_offsets(u_int8_t af)
vfree(xt[af].compat_tab); vfree(xt[af].compat_tab);
xt[af].compat_tab = NULL; xt[af].compat_tab = NULL;
xt[af].number = 0; xt[af].number = 0;
xt[af].cur = 0;
} }
} }
EXPORT_SYMBOL_GPL(xt_compat_flush_offsets); EXPORT_SYMBOL_GPL(xt_compat_flush_offsets);
...@@ -473,8 +474,7 @@ int xt_compat_calc_jump(u_int8_t af, unsigned int offset) ...@@ -473,8 +474,7 @@ int xt_compat_calc_jump(u_int8_t af, unsigned int offset)
else else
return mid ? tmp[mid - 1].delta : 0; return mid ? tmp[mid - 1].delta : 0;
} }
WARN_ON_ONCE(1); return left ? tmp[left - 1].delta : 0;
return 0;
} }
EXPORT_SYMBOL_GPL(xt_compat_calc_jump); EXPORT_SYMBOL_GPL(xt_compat_calc_jump);
......
...@@ -99,7 +99,7 @@ tos_tg6(struct sk_buff *skb, const struct xt_action_param *par) ...@@ -99,7 +99,7 @@ tos_tg6(struct sk_buff *skb, const struct xt_action_param *par)
u_int8_t orig, nv; u_int8_t orig, nv;
orig = ipv6_get_dsfield(iph); orig = ipv6_get_dsfield(iph);
nv = (orig & info->tos_mask) ^ info->tos_value; nv = (orig & ~info->tos_mask) ^ info->tos_value;
if (orig != nv) { if (orig != nv) {
if (!skb_make_writable(skb, sizeof(struct iphdr))) if (!skb_make_writable(skb, sizeof(struct iphdr)))
......
...@@ -272,11 +272,6 @@ static int conntrack_mt_check(const struct xt_mtchk_param *par) ...@@ -272,11 +272,6 @@ static int conntrack_mt_check(const struct xt_mtchk_param *par)
{ {
int ret; int ret;
if (strcmp(par->table, "raw") == 0) {
pr_info("state is undetermined at the time of raw table\n");
return -EINVAL;
}
ret = nf_ct_l3proto_try_module_get(par->family); ret = nf_ct_l3proto_try_module_get(par->family);
if (ret < 0) if (ret < 0)
pr_info("cannot load conntrack support for proto=%u\n", pr_info("cannot load conntrack support for proto=%u\n",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment