devlink: Add health recover notifications on devlink flows

Devlink health recover notifications were added only on driver direct
updates of health_state through devlink_health_reporter_state_update().
Add notifications on updates of health_state by devlink flows of report
and recover.

Moved functions devlink_nl_health_reporter_fill() and
devlink_recover_notify() to avoid forward declaration.

Fixes: 97ff3bd37f ("devlink: add devink notification when reporter update health state")
Signed-off-by: Moshe Shemesh <moshe@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Moshe Shemesh 2020-01-23 19:57:13 +02:00 committed by David S. Miller
parent 53c6770095
commit 6ec8b6cd79

View File

@ -4843,180 +4843,6 @@ devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
}
EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
void
devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
{
reporter->recovery_count++;
reporter->last_recovery_ts = jiffies;
}
EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
static int
devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
void *priv_ctx, struct netlink_ext_ack *extack)
{
int err;
if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
return 0;
if (!reporter->ops->recover)
return -EOPNOTSUPP;
err = reporter->ops->recover(reporter, priv_ctx, extack);
if (err)
return err;
devlink_health_reporter_recovery_done(reporter);
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
return 0;
}
static void
devlink_health_dump_clear(struct devlink_health_reporter *reporter)
{
if (!reporter->dump_fmsg)
return;
devlink_fmsg_free(reporter->dump_fmsg);
reporter->dump_fmsg = NULL;
}
static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
void *priv_ctx,
struct netlink_ext_ack *extack)
{
int err;
if (!reporter->ops->dump)
return 0;
if (reporter->dump_fmsg)
return 0;
reporter->dump_fmsg = devlink_fmsg_alloc();
if (!reporter->dump_fmsg) {
err = -ENOMEM;
return err;
}
err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
if (err)
goto dump_err;
err = reporter->ops->dump(reporter, reporter->dump_fmsg,
priv_ctx, extack);
if (err)
goto dump_err;
err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
if (err)
goto dump_err;
reporter->dump_ts = jiffies;
reporter->dump_real_ts = ktime_get_real_ns();
return 0;
dump_err:
devlink_health_dump_clear(reporter);
return err;
}
int devlink_health_report(struct devlink_health_reporter *reporter,
const char *msg, void *priv_ctx)
{
enum devlink_health_reporter_state prev_health_state;
struct devlink *devlink = reporter->devlink;
/* write a log message of the current error */
WARN_ON(!msg);
trace_devlink_health_report(devlink, reporter->ops->name, msg);
reporter->error_count++;
prev_health_state = reporter->health_state;
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
/* abort if the previous error wasn't recovered */
if (reporter->auto_recover &&
(prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
jiffies - reporter->last_recovery_ts <
msecs_to_jiffies(reporter->graceful_period))) {
trace_devlink_health_recover_aborted(devlink,
reporter->ops->name,
reporter->health_state,
jiffies -
reporter->last_recovery_ts);
return -ECANCELED;
}
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
mutex_lock(&reporter->dump_lock);
/* store current dump of current error, for later analysis */
devlink_health_do_dump(reporter, priv_ctx, NULL);
mutex_unlock(&reporter->dump_lock);
if (reporter->auto_recover)
return devlink_health_reporter_recover(reporter,
priv_ctx, NULL);
return 0;
}
EXPORT_SYMBOL_GPL(devlink_health_report);
static struct devlink_health_reporter *
devlink_health_reporter_get_from_attrs(struct devlink *devlink,
struct nlattr **attrs)
{
struct devlink_health_reporter *reporter;
char *reporter_name;
if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
return NULL;
reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
mutex_lock(&devlink->reporters_lock);
reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
if (reporter)
refcount_inc(&reporter->refcount);
mutex_unlock(&devlink->reporters_lock);
return reporter;
}
static struct devlink_health_reporter *
devlink_health_reporter_get_from_info(struct devlink *devlink,
struct genl_info *info)
{
return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
}
static struct devlink_health_reporter *
devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
{
const struct genl_dumpit_info *info = genl_dumpit_info(cb);
struct devlink_health_reporter *reporter;
struct nlattr **attrs = info->attrs;
struct devlink *devlink;
mutex_lock(&devlink_mutex);
devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
if (IS_ERR(devlink))
goto unlock;
reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
mutex_unlock(&devlink_mutex);
return reporter;
unlock:
mutex_unlock(&devlink_mutex);
return NULL;
}
static void
devlink_health_reporter_put(struct devlink_health_reporter *reporter)
{
refcount_dec(&reporter->refcount);
}
static int
devlink_nl_health_reporter_fill(struct sk_buff *msg,
struct devlink *devlink,
@ -5104,6 +4930,182 @@ static void devlink_recover_notify(struct devlink_health_reporter *reporter,
msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
}
void
devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
{
reporter->recovery_count++;
reporter->last_recovery_ts = jiffies;
}
EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
static int
devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
void *priv_ctx, struct netlink_ext_ack *extack)
{
int err;
if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
return 0;
if (!reporter->ops->recover)
return -EOPNOTSUPP;
err = reporter->ops->recover(reporter, priv_ctx, extack);
if (err)
return err;
devlink_health_reporter_recovery_done(reporter);
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
return 0;
}
static void
devlink_health_dump_clear(struct devlink_health_reporter *reporter)
{
if (!reporter->dump_fmsg)
return;
devlink_fmsg_free(reporter->dump_fmsg);
reporter->dump_fmsg = NULL;
}
static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
void *priv_ctx,
struct netlink_ext_ack *extack)
{
int err;
if (!reporter->ops->dump)
return 0;
if (reporter->dump_fmsg)
return 0;
reporter->dump_fmsg = devlink_fmsg_alloc();
if (!reporter->dump_fmsg) {
err = -ENOMEM;
return err;
}
err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
if (err)
goto dump_err;
err = reporter->ops->dump(reporter, reporter->dump_fmsg,
priv_ctx, extack);
if (err)
goto dump_err;
err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
if (err)
goto dump_err;
reporter->dump_ts = jiffies;
reporter->dump_real_ts = ktime_get_real_ns();
return 0;
dump_err:
devlink_health_dump_clear(reporter);
return err;
}
int devlink_health_report(struct devlink_health_reporter *reporter,
const char *msg, void *priv_ctx)
{
enum devlink_health_reporter_state prev_health_state;
struct devlink *devlink = reporter->devlink;
/* write a log message of the current error */
WARN_ON(!msg);
trace_devlink_health_report(devlink, reporter->ops->name, msg);
reporter->error_count++;
prev_health_state = reporter->health_state;
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
/* abort if the previous error wasn't recovered */
if (reporter->auto_recover &&
(prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
jiffies - reporter->last_recovery_ts <
msecs_to_jiffies(reporter->graceful_period))) {
trace_devlink_health_recover_aborted(devlink,
reporter->ops->name,
reporter->health_state,
jiffies -
reporter->last_recovery_ts);
return -ECANCELED;
}
reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
mutex_lock(&reporter->dump_lock);
/* store current dump of current error, for later analysis */
devlink_health_do_dump(reporter, priv_ctx, NULL);
mutex_unlock(&reporter->dump_lock);
if (reporter->auto_recover)
return devlink_health_reporter_recover(reporter,
priv_ctx, NULL);
return 0;
}
EXPORT_SYMBOL_GPL(devlink_health_report);
static struct devlink_health_reporter *
devlink_health_reporter_get_from_attrs(struct devlink *devlink,
struct nlattr **attrs)
{
struct devlink_health_reporter *reporter;
char *reporter_name;
if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
return NULL;
reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
mutex_lock(&devlink->reporters_lock);
reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
if (reporter)
refcount_inc(&reporter->refcount);
mutex_unlock(&devlink->reporters_lock);
return reporter;
}
static struct devlink_health_reporter *
devlink_health_reporter_get_from_info(struct devlink *devlink,
struct genl_info *info)
{
return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
}
static struct devlink_health_reporter *
devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
{
const struct genl_dumpit_info *info = genl_dumpit_info(cb);
struct devlink_health_reporter *reporter;
struct nlattr **attrs = info->attrs;
struct devlink *devlink;
mutex_lock(&devlink_mutex);
devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
if (IS_ERR(devlink))
goto unlock;
reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
mutex_unlock(&devlink_mutex);
return reporter;
unlock:
mutex_unlock(&devlink_mutex);
return NULL;
}
static void
devlink_health_reporter_put(struct devlink_health_reporter *reporter)
{
refcount_dec(&reporter->refcount);
}
void
devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
enum devlink_health_reporter_state state)