[Powerpc-utils-devel] [PATCH v2] drmgr: Update drmgr for in-kernel memory hotplug

[Nathan F. <nf...@li...>]

To further update the hotplug capabilities of the PowerVM and PowerKVM
platforms, the hotplug infrastructure is being updated to provide an
in-kernel implementation. In order to take advantage of this on the
PowerVM platforms the kernel will provide a new /sys/kernel/dlpar
file that we can use to initiate hotplug events in the kernel.

Additionally, the new in-kernel infrastructure uses rtas hotplug events
to communicate hotplug actions.

This patch provides a common routine to create these rtas hotplug
events and write them to the sysfs file. The patch also provides the
infrastructure to do memory hotplug in the kernel in conjunction
with the in-kernel memory implementation submitted upstream.

Signed-off-by: Nathan Fontenot <nf...@li...>
---
 src/drmgr/common.c          |  132 +++++++++++++++++++++++++++++++++++++++++++
 src/drmgr/dr.h              |   14 +++++
 src/drmgr/drslot_chrp_mem.c |   54 +++++++++++++++---
 3 files changed, 192 insertions(+), 8 deletions(-)

diff --git a/src/drmgr/common.c b/src/drmgr/common.c
index 0cb621d..78a1c0f 100644
--- a/src/drmgr/common.c
+++ b/src/drmgr/common.c
@@ -28,6 +28,7 @@ char *remove_slot_fname = REMOVE_SLOT_FNAME;
 #define DR_LOG_PATH0	"/var/log/drmgr.0"
 
 #define LPARCFG_PATH	"/proc/ppc64/lparcfg"
+#define RTAS_HP_PATH	"/sys/kernel/dlpar"
 
 static int dr_lock_fd = 0;
 static long dr_timeout;
@@ -1372,3 +1373,134 @@ int ams_balloon_active(void)
 
 	return !is_inactive;
 }
+
+/**
+ * kernel_hotplug_rtas_exists
+ * @brief Determine if kernel interface to do hotplug via RTAS events exists
+ *
+ * @returns 1 if it exists, 0 otherwise
+ */
+int kernel_hotplug_rtas_exists(void)
+{
+	struct stat sbuf;
+
+	if (!stat(RTAS_HP_PATH, &sbuf))
+		return 1;
+
+	return 0;
+}
+
+/**
+ * send_rtas_event
+ * @brief Write the supplied rtas event to /proc to initiatye a hotplug event
+ *
+ * @param
+ *
+ */
+int send_rtas_event(struct hp_rtas_event *hp_event)
+{
+	struct rtas_event_hdr_raw *hdr = &hp_event->hdr;
+	int rc = -1;
+	int fd, length;
+
+	fd = open(RTAS_HP_PATH, O_WRONLY);
+	if (fd <= 0) {
+		say(ERROR, "Could not open \"%s\"\n%s\n", RTAS_HP_PATH,
+		    strerror(errno));
+		return -1;
+	}
+
+	length = be32toh(hdr->ext_log_length) + sizeof(*hdr);
+	say(DEBUG, "Writing hotplug rtas event to %s...", RTAS_HP_PATH);
+	rc = write(fd, hp_event, length);
+	close(fd);
+
+	if (rc != length) {
+		say(DEBUG, "%s:\n%s\n", "fail", strerror(errno));
+	} else {
+		say(DEBUG, "Success\n");
+		rc = 0;
+	}
+
+	return rc;
+}
+
+struct hp_rtas_event *alloc_hp_rtas_event(struct options *opts, int type)
+{
+	struct hp_rtas_event *hp_event;
+	struct rtas_event_hdr_raw *hdr;
+	struct rtas_event_exthdr_raw *exthdr;
+	struct rtas_priv_hdr_scn_raw *privhdr;
+	struct rtas_usr_hdr_scn_raw *usrhdr;
+	struct rtas_hotplug_scn_raw *hpscn;
+	int event_size;
+
+	event_size = sizeof(*hp_event);
+	if (opts->usr_drc_name)
+		event_size += strlen(opts->usr_drc_name);
+
+	hp_event = zalloc(event_size);
+	if (!hp_event)
+		return NULL;
+
+	hdr = &hp_event->hdr;
+	hdr->version = 6;
+	hdr->data1 = (RTAS_HDR_SEV_EVENT << 5)
+		     || (RTAS_HDR_DISP_NOT_RECOVERED << 3) || 1;
+
+	hdr->data2 = RTAS_HDR_INIT_HOT_PLUG << 4;
+	hdr->type = RTAS_HDR_TYPE_HOTPLUG;
+	hdr->ext_log_length = htobe32(event_size - sizeof(*hdr));
+
+	exthdr = &hp_event->exthdr;
+	/* Set valid, newlog, and bigendian bits */
+	exthdr->data1 = 0x86;
+
+	/* Set power_pc bit and format (14) bits */
+	exthdr->data3 = 0x8e;
+
+	hp_event->ibm = htobe32((('I' << 24) | ('B' << 16) | ('M' << 8)));
+	/* exthdr->format_type = HP? */
+	/* TODO: Fill in rtas date/time */
+
+	privhdr = &hp_event->privhdr;
+	privhdr->v6hdr.id[0] = 'P';
+	privhdr->v6hdr.id[1] = 'H';
+	privhdr->v6hdr.length = htobe16(sizeof(*privhdr));
+	privhdr->v6hdr.version = 1;
+	/* TODO: Fille in rtas date/time */
+	privhdr->scn_count = 1;
+
+	usrhdr = &hp_event->usrhdr;
+	usrhdr->v6hdr.id[0] = 'U';
+	usrhdr->v6hdr.id[1] = 'H';
+	usrhdr->v6hdr.length = htobe16(sizeof(*usrhdr));
+
+	hpscn = &hp_event->hpscn;
+	hpscn->v6hdr.id[0] = 'H';
+	hpscn->v6hdr.id[1] = 'P';
+	hpscn->type = type;
+
+	if (opts->action == ADD)
+		hpscn->action = RTAS_HP_ACTION_ADD;
+	else
+		hpscn->action = RTAS_HP_ACTION_REMOVE;
+
+	if (opts->usr_drc_index) {
+		hpscn->v6hdr.length = htobe16(sizeof(*hpscn));
+		hpscn->identifier = RTAS_HP_ID_DRC_INDEX;
+		hpscn->u1.drc_index = htobe32(opts->usr_drc_index);
+	} else if (opts->usr_drc_name) {
+		hpscn->v6hdr.length = htobe16(sizeof(*hpscn)
+					      + strlen(opts->usr_drc_name));
+		hpscn->identifier = RTAS_HP_ID_DRC_NAME;
+		memcpy(hpscn->u1.drc_name, opts->usr_drc_name,
+		       strlen(opts->usr_drc_name));
+	} else {
+		hpscn->v6hdr.length = htobe16(sizeof(*hpscn));
+		hpscn->identifier = RTAS_HP_ID_DRC_COUNT;
+		hpscn->u1.count = htobe32(opts->quantity);
+	}
+
+	return hp_event;
+}
diff --git a/src/drmgr/dr.h b/src/drmgr/dr.h
index 9ae7a82..dfd46fc 100644
--- a/src/drmgr/dr.h
+++ b/src/drmgr/dr.h
@@ -12,6 +12,7 @@
 #include <nl_types.h>
 #include <unistd.h>
 #include <stdarg.h>
+#include <librtasevent.h>
 #include "rtas_calls.h"
 #include "drpci.h"
 
@@ -68,6 +69,16 @@ struct options {
 
 enum say_level { ERROR = 1, WARN, INFO, DEBUG};
 
+/* Skeleton hotplug RTAS Event */
+struct hp_rtas_event {
+	struct rtas_event_hdr_raw hdr;
+	struct rtas_event_exthdr_raw exthdr;
+	uint32_t ibm;
+	struct rtas_priv_hdr_scn_raw privhdr;
+	struct rtas_usr_hdr_scn_raw usrhdr;
+	struct rtas_hotplug_scn_raw hpscn;
+};
+
 /* The follwing are defined in common.c */
 int say(enum say_level, char *, ...);
 void report_unknown_error(char *, int);
@@ -78,6 +89,9 @@ int drmgr_timed_out(void);
 int dr_lock(void);
 int dr_unlock(void);
 int valid_platform(const char *);
+int kernel_hotplug_rtas_exists(void);
+int send_rtas_event(struct hp_rtas_event *);
+struct hp_rtas_event *alloc_hp_rtas_event(struct options *, int);
 
 void free_of_node(struct of_node *);
 int add_device_tree_nodes(char *, struct of_node *);
diff --git a/src/drmgr/drslot_chrp_mem.c b/src/drmgr/drslot_chrp_mem.c
index d6ece97..062a10d 100644
--- a/src/drmgr/drslot_chrp_mem.c
+++ b/src/drmgr/drslot_chrp_mem.c
@@ -1015,6 +1015,41 @@ mem_add(struct options *opts)
 }
 
 /**
+ * mem_add_rtas
+ * @brief Add memory via the kernel rtas event interface
+ *
+ * @param opts user options
+ * @returns 0 on success, !0 otherwise
+ */
+static int mem_hp_rtas(struct options *opts)
+{
+	struct hp_rtas_event *hp_event;
+	int rc;
+
+	hp_event = alloc_hp_rtas_event(opts, RTAS_HP_TYPE_MEMORY);
+	if (!hp_event)
+		return -1;
+
+	rc = send_rtas_event(hp_event);
+	if (rc) {
+		say(ERROR, "Hotplug Failed\n");
+		printf("DR_TOTAL_RESOURCES=0\n");
+	} else {
+		int lmbs;
+
+		if (opts->usr_drc_index)
+			lmbs = 1;
+		else
+			lmbs = opts->quantity;
+
+		printf("DR_TOTAL_RESOURCES=%d\n", lmbs);
+	}
+
+	free(hp_event);
+	return rc;
+}
+
+/**
  * remove_lmbs
  *
  * @param nr_lmbs
@@ -1251,14 +1286,17 @@ drslot_chrp_mem(struct options *opts)
 	if (opts->usr_drc_name)
 		opts->quantity = 1;
 
-	switch (opts->action) {
-	    case ADD:
-		rc = mem_add(opts);
-		break;
-
-	    case REMOVE:
-		rc = mem_remove(opts);
-		break;
+	if (kernel_hotplug_rtas_exists()) {
+		rc = mem_hp_rtas(opts);
+	} else {
+		switch (opts->action) {
+		case ADD:
+			rc = mem_add(opts);
+			break;
+		case REMOVE:
+			rc = mem_remove(opts);
+			break;
+		}
 	}
 
 	return rc;