Excerpts from Haren Myneni's message of January 22, 2022 5:56 am: > > VAS windows can be closed in the hypervisor due to lost credits > when the core is removed. If these credits are available later > for core add, reopen these windows and set them active. When the > kernel sees page fault on the paste address, it creates new mapping > on the new paste address. Then the user space can continue to use > these windows and send HW compression requests to NX successfully.
Any reason to put this before the close windows patch? It would be more logical to put it afterwards AFAIKS. > > Signed-off-by: Haren Myneni <ha...@linux.ibm.com> > --- > arch/powerpc/include/asm/vas.h | 16 +++ > arch/powerpc/platforms/book3s/vas-api.c | 1 + > arch/powerpc/platforms/pseries/vas.c | 144 ++++++++++++++++++++++++ > arch/powerpc/platforms/pseries/vas.h | 8 +- > 4 files changed, 163 insertions(+), 6 deletions(-) > > diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h > index 57573d9c1e09..f1efe86563cc 100644 > --- a/arch/powerpc/include/asm/vas.h > +++ b/arch/powerpc/include/asm/vas.h > @@ -29,6 +29,19 @@ > #define VAS_THRESH_FIFO_GT_QTR_FULL 2 > #define VAS_THRESH_FIFO_GT_EIGHTH_FULL 3 > > +/* > + * VAS window status > + */ > +#define VAS_WIN_ACTIVE 0x0 /* Used in platform independent > */ > + /* vas mmap() */ > +/* The hypervisor returns these values */ > +#define VAS_WIN_CLOSED 0x00000001 > +#define VAS_WIN_INACTIVE 0x00000002 /* Inactive due to HW failure */ > +#define VAS_WIN_MOD_IN_PROCESS 0x00000003 /* Process of being > modified, */ While you're moving these and adding a comment, it would be good to list what hcalls they are relevant to. H_QUERY_VAS_WINDOW (which is not used anywhere yet?) These are also a 1-byte field, so '0x00', '0x01' etc would be more appropriate. > + /* deallocated, or quiesced */ > +/* Linux status bits */ > +#define VAS_WIN_NO_CRED_CLOSE 0x00000004 /* Window is closed due to */ > + /* lost credit */ This is mixing a user defined bit field with hcall API value field. You also AFAIKS as yet don't fill in the hypervisor status anywhere. I would make this it's own field entirely. A boolean would be nice, if possible. > /* > * Get/Set bit fields > */ > @@ -59,6 +72,8 @@ struct vas_user_win_ref { > struct pid *pid; /* PID of owner */ > struct pid *tgid; /* Thread group ID of owner */ > struct mm_struct *mm; /* Linux process mm_struct */ > + struct mutex mmap_mutex; /* protects paste address mmap() */ > + /* with DLPAR close/open windows */ > }; > > /* > @@ -67,6 +82,7 @@ struct vas_user_win_ref { > struct vas_window { > u32 winid; > u32 wcreds_max; /* Window credits */ > + u32 status; > enum vas_cop_type cop; > struct vas_user_win_ref task_ref; > char *dbgname; > diff --git a/arch/powerpc/platforms/book3s/vas-api.c > b/arch/powerpc/platforms/book3s/vas-api.c > index 4d82c92ddd52..2b0ced611f32 100644 > --- a/arch/powerpc/platforms/book3s/vas-api.c > +++ b/arch/powerpc/platforms/book3s/vas-api.c > @@ -316,6 +316,7 @@ static int coproc_ioc_tx_win_open(struct file *fp, > unsigned long arg) > return PTR_ERR(txwin); > } > > + mutex_init(&txwin->task_ref.mmap_mutex); > cp_inst->txwin = txwin; > > return 0; > diff --git a/arch/powerpc/platforms/pseries/vas.c > b/arch/powerpc/platforms/pseries/vas.c > index 2ef56157634f..d9ff73d7704d 100644 > --- a/arch/powerpc/platforms/pseries/vas.c > +++ b/arch/powerpc/platforms/pseries/vas.c > @@ -501,6 +501,7 @@ static int __init get_vas_capabilities(u8 feat, enum > vas_cop_feat_type type, > memset(vcaps, 0, sizeof(*vcaps)); > INIT_LIST_HEAD(&vcaps->list); > > + vcaps->feat = feat; > caps = &vcaps->caps; > > rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, feat, > @@ -539,6 +540,145 @@ static int __init get_vas_capabilities(u8 feat, enum > vas_cop_feat_type type, > return 0; > } > > +/* > + * VAS windows can be closed due to lost credits when the core is > + * removed. So reopen them if credits are available due to DLPAR > + * core add and set the window active status. When NX sees the page > + * fault on the unmapped paste address, the kernel handles the fault > + * by setting the remapping to new paste address if the window is > + * active. > + */ > +static int reconfig_open_windows(struct vas_caps *vcaps, int creds) > +{ > + long domain[PLPAR_HCALL9_BUFSIZE] = {VAS_DEFAULT_DOMAIN_ID}; > + struct vas_cop_feat_caps *caps = &vcaps->caps; > + struct pseries_vas_window *win = NULL, *tmp; > + int rc, mv_ents = 0; > + > + /* > + * Nothing to do if there are no closed windows. > + */ > + if (!vcaps->close_wins) > + return 0; > + > + /* > + * For the core removal, the hypervisor reduces the credits > + * assigned to the LPAR and the kernel closes VAS windows > + * in the hypervisor depends on reduced credits. The kernel > + * uses LIFO (the last windows that are opened will be closed > + * first) and expects to open in the same order when credits > + * are available. > + * For example, 40 windows are closed when the LPAR lost 2 cores > + * (dedicated). If 1 core is added, this LPAR can have 20 more > + * credits. It means the kernel can reopen 20 windows. So move > + * 20 entries in the VAS windows lost and reopen next 20 windows. > + */ > + if (vcaps->close_wins > creds) > + mv_ents = vcaps->close_wins - creds; > + > + list_for_each_entry_safe(win, tmp, &vcaps->list, win_list) { > + if (!mv_ents) > + break; > + > + mv_ents--; > + } > + > + list_for_each_entry_safe_from(win, tmp, &vcaps->list, win_list) { > + /* > + * Nothing to do on this window if it is not closed > + * with VAS_WIN_NO_CRED_CLOSE > + */ > + if (!(win->vas_win.status & VAS_WIN_NO_CRED_CLOSE)) > + continue; > + > + rc = allocate_setup_window(win, (u64 *)&domain[0], > + caps->win_type); > + if (rc) > + return rc; > + > + rc = h_modify_vas_window(win); > + if (rc) > + goto out; > + > + mutex_lock(&win->vas_win.task_ref.mmap_mutex); > + /* > + * Set window status to active > + */ > + win->vas_win.status &= ~VAS_WIN_NO_CRED_CLOSE; > + mutex_unlock(&win->vas_win.task_ref.mmap_mutex); What's the mutex protecting? If status can change, what happens if it changed between checking it above and this point? > + win->win_type = caps->win_type; > + if (!--vcaps->close_wins) > + break; > + } > + > + return 0; > +out: > + /* > + * Window modify HCALL failed. So close the window to the > + * hypervisor and return. > + */ > + free_irq_setup(win); > + h_deallocate_vas_window(win->vas_win.winid); > + return rc; > +} > + > +/* > + * Get new VAS capabilities when the core add/removal configuration > + * changes. Reconfig window configurations based on the credits > + * availability from this new capabilities. > + */ > +static int vas_reconfig_capabilties(u8 type) > +{ > + struct hv_vas_cop_feat_caps *hv_caps; > + struct vas_cop_feat_caps *caps; > + int lpar_creds, new_creds; > + struct vas_caps *vcaps; > + int rc = 0; > + > + if (type >= VAS_MAX_FEAT_TYPE) { > + pr_err("Invalid credit type %d\n", type); > + return -EINVAL; > + } > + > + vcaps = &vascaps[type]; > + caps = &vcaps->caps; > + > + hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); > + if (!hv_caps) > + return -ENOMEM; > + > + mutex_lock(&vas_pseries_mutex); > + rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, vcaps->feat, > + (u64)virt_to_phys(hv_caps)); > + if (rc) > + goto out; > + > + new_creds = be16_to_cpu(hv_caps->target_lpar_creds); > + > + lpar_creds = atomic_read(&caps->target_creds); NBD but it would be slightly clearer that new_creds is the absolute value not the number of new credits if you call these two new_nr_creds and old_nr_creds or prev_nr_creds. > + > + atomic_set(&caps->target_creds, new_creds); > + /* > + * The total number of available credits may be decreased or > + * inceased with DLPAR operation. Means some windows have to be > + * closed / reopened. Hold the vas_pseries_mutex so that the > + * the user space can not open new windows. > + */ > + if (lpar_creds < new_creds) { > + /* > + * If the existing target credits is less than the new > + * target, reopen windows if they are closed due to > + * the previous DLPAR (core removal). > + */ > + rc = reconfig_open_windows(vcaps, new_creds - lpar_creds); > + } > + > +out: > + mutex_unlock(&vas_pseries_mutex); > + kfree(hv_caps); > + return rc; > +} > + > /* > * Total number of default credits available (target_credits) > * in LPAR depends on number of cores configured. It varies based on > @@ -565,6 +705,10 @@ static int pseries_vas_notifier(struct notifier_block > *nb, > if (!intserv) > return NOTIFY_OK; > > + rc = vas_reconfig_capabilties(VAS_GZIP_DEF_FEAT_TYPE); > + if (rc) > + pr_err("Failed reconfig VAS capabilities with DLPAR\n"); > + > return rc; > } > > diff --git a/arch/powerpc/platforms/pseries/vas.h > b/arch/powerpc/platforms/pseries/vas.h > index 0538760d13be..45b62565955b 100644 > --- a/arch/powerpc/platforms/pseries/vas.h > +++ b/arch/powerpc/platforms/pseries/vas.h > @@ -21,12 +21,6 @@ > #define VAS_MOD_WIN_FLAGS (VAS_MOD_WIN_JOBS_KILL | VAS_MOD_WIN_DR | \ > VAS_MOD_WIN_PR | VAS_MOD_WIN_SF) > > -#define VAS_WIN_ACTIVE 0x0 > -#define VAS_WIN_CLOSED 0x1 > -#define VAS_WIN_INACTIVE 0x2 /* Inactive due to HW failure */ > -/* Process of being modified, deallocated, or quiesced */ > -#define VAS_WIN_MOD_IN_PROCESS 0x3 > - > #define VAS_COPY_PASTE_USER_MODE 0x00000001 > #define VAS_COP_OP_USER_MODE 0x00000010 > > @@ -84,6 +78,8 @@ struct vas_cop_feat_caps { > struct vas_caps { > struct vas_cop_feat_caps caps; > struct list_head list; /* List of open windows */ > + int close_wins; /* closed windows in the hypervisor for DLPAR */ 'nr_closed_wins' This does not look like a capability. Although I guess neither do some of the fields (used_creds) in vas_cop_feat_caps. Why not at least put this there with the rest of those credits fields? Thanks, Nick > + u8 feat; /* Feature type */ > }; > > /* > -- > 2.27.0 > > >