/* SPDX-License-Identifier: GPL-2.0-or-later */
/******************************************************************************
 * arch/x86/paging.c
 *
 * x86 specific paging support
 * Copyright (c) 2007 Advanced Micro Devices (Wei Huang)
 * Copyright (c) 2007 XenSource Inc.
 */

#include <xen/init.h>
#include <xen/guest_access.h>
#include <xen/hypercall.h>
#include <asm/paging.h>
#include <asm/shadow.h>
#include <asm/p2m.h>
#include <asm/hap.h>
#include <asm/event.h>
#include <asm/hvm/nestedhvm.h>
#include <xen/numa.h>
#include <xsm/xsm.h>
#include <public/sched.h> /* SHUTDOWN_suspend */

#include "mm-locks.h"

/* Printouts */
#define PAGING_PRINTK(_f, _a...)                                     \
    debugtrace_printk("pg: %s(): " _f, __func__, ##_a)
#define PAGING_ERROR(_f, _a...)                                      \
    printk("pg error: %s(): " _f, __func__, ##_a)
#define PAGING_DEBUG(flag, _f, _a...)                                \
    do {                                                             \
        if (PAGING_DEBUG_ ## flag)                                   \
            debugtrace_printk("pgdebug: %s(): " _f, __func__, ##_a); \
    } while (0)

/* Per-CPU variable for enforcing the lock ordering */
DEFINE_PER_CPU(int, mm_lock_level);

#if PG_log_dirty

/************************************************/
/*              LOG DIRTY SUPPORT               */
/************************************************/

static mfn_t paging_new_log_dirty_page(struct domain *d)
{
    struct page_info *page;

    page = d->arch.paging.alloc_page(d);
    if ( unlikely(page == NULL) )
    {
        d->arch.paging.log_dirty.failed_allocs++;
        return INVALID_MFN;
    }

    d->arch.paging.log_dirty.allocs++;

    return page_to_mfn(page);
}

/* Alloc and init a new leaf node */
static mfn_t paging_new_log_dirty_leaf(struct domain *d)
{
    mfn_t mfn = paging_new_log_dirty_page(d);

    if ( !mfn_eq(mfn, INVALID_MFN) )
        clear_domain_page(mfn);

    return mfn;
}

/* Alloc and init a new non-leaf node */
static mfn_t paging_new_log_dirty_node(struct domain *d)
{
    mfn_t mfn = paging_new_log_dirty_page(d);

    if ( !mfn_eq(mfn, INVALID_MFN) )
    {
        int i;
        mfn_t *node = map_domain_page(mfn);
        for ( i = 0; i < LOGDIRTY_NODE_ENTRIES; i++ )
            node[i] = INVALID_MFN;
        unmap_domain_page(node);
    }
    return mfn;
}

/* get the top of the log-dirty bitmap trie */
static mfn_t *paging_map_log_dirty_bitmap(struct domain *d)
{
    if ( likely(!mfn_eq(d->arch.paging.log_dirty.top, INVALID_MFN)) )
        return map_domain_page(d->arch.paging.log_dirty.top);
    return NULL;
}

static void paging_free_log_dirty_page(struct domain *d, mfn_t mfn)
{
    d->arch.paging.log_dirty.allocs--;
    d->arch.paging.free_page(d, mfn_to_page(mfn));
}

static int paging_free_log_dirty_bitmap(struct domain *d, int rc)
{
    mfn_t *l4, *l3, *l2;
    int i4, i3, i2;

    paging_lock(d);

    if ( mfn_eq(d->arch.paging.log_dirty.top, INVALID_MFN) )
    {
        paging_unlock(d);
        return 0;
    }

    if ( !d->arch.paging.preempt.dom )
    {
        memset(&d->arch.paging.preempt.log_dirty, 0,
               sizeof(d->arch.paging.preempt.log_dirty));
        ASSERT(rc <= 0);
        d->arch.paging.preempt.log_dirty.done = -rc;
    }
    else if ( d->arch.paging.preempt.dom != current->domain ||
              d->arch.paging.preempt.op != XEN_DOMCTL_SHADOW_OP_OFF )
    {
        paging_unlock(d);
        return -EBUSY;
    }

    l4 = map_domain_page(d->arch.paging.log_dirty.top);
    i4 = d->arch.paging.preempt.log_dirty.i4;
    i3 = d->arch.paging.preempt.log_dirty.i3;
    rc = 0;

    for ( ; i4 < LOGDIRTY_NODE_ENTRIES; i4++, i3 = 0 )
    {
        if ( mfn_eq(l4[i4], INVALID_MFN) )
            continue;

        l3 = map_domain_page(l4[i4]);

        for ( ; i3 < LOGDIRTY_NODE_ENTRIES; i3++ )
        {
            if ( mfn_eq(l3[i3], INVALID_MFN) )
                continue;

            l2 = map_domain_page(l3[i3]);

            for ( i2 = 0; i2 < LOGDIRTY_NODE_ENTRIES; i2++ )
                if ( !mfn_eq(l2[i2], INVALID_MFN) )
                    paging_free_log_dirty_page(d, l2[i2]);

            unmap_domain_page(l2);
            paging_free_log_dirty_page(d, l3[i3]);
            l3[i3] = INVALID_MFN;

            if ( i3 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() )
            {
                d->arch.paging.preempt.log_dirty.i3 = i3 + 1;
                d->arch.paging.preempt.log_dirty.i4 = i4;
                rc = -ERESTART;
                break;
            }
        }

        unmap_domain_page(l3);
        if ( rc )
            break;
        paging_free_log_dirty_page(d, l4[i4]);
        l4[i4] = INVALID_MFN;

        if ( i4 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() )
        {
            d->arch.paging.preempt.log_dirty.i3 = 0;
            d->arch.paging.preempt.log_dirty.i4 = i4 + 1;
            rc = -ERESTART;
            break;
        }
    }

    unmap_domain_page(l4);

    if ( !rc )
    {
        paging_free_log_dirty_page(d, d->arch.paging.log_dirty.top);
        d->arch.paging.log_dirty.top = INVALID_MFN;

        ASSERT(d->arch.paging.log_dirty.allocs == 0);
        d->arch.paging.log_dirty.failed_allocs = 0;

        rc = -d->arch.paging.preempt.log_dirty.done;
        d->arch.paging.preempt.dom = NULL;
    }
    else
    {
        d->arch.paging.preempt.dom = current->domain;
        d->arch.paging.preempt.op = XEN_DOMCTL_SHADOW_OP_OFF;
    }

    paging_unlock(d);

    return rc;
}

static int paging_log_dirty_enable(struct domain *d)
{
    int ret;

    if ( has_arch_pdevs(d) )
    {
        /*
         * Refuse to turn on global log-dirty mode
         * if the domain is sharing the P2M with the IOMMU.
         */
        return -EINVAL;
    }

    if ( paging_mode_log_dirty(d) )
        return -EINVAL;

    domain_pause(d);
    ret = d->arch.paging.log_dirty.ops->enable(d);
    domain_unpause(d);

    return ret;
}

static int paging_log_dirty_disable(struct domain *d, bool resuming)
{
    int ret = 1;

    if ( !resuming )
    {
        domain_pause(d);
        /* Safe because the domain is paused. */
        if ( paging_mode_log_dirty(d) )
        {
            ret = d->arch.paging.log_dirty.ops->disable(d);
            ASSERT(ret <= 0);
        }
    }

    ret = paging_free_log_dirty_bitmap(d, ret);
    if ( ret == -ERESTART )
        return ret;

    domain_unpause(d);

    return ret;
}

/* Mark a page as dirty, with taking guest pfn as parameter */
static void set_pfn_logdirty(struct domain *d, pfn_t pfn, bool dirty)
{
    bool changed;
    mfn_t mfn, *l4, *l3, *l2;
    unsigned long *l1;
    unsigned int i1, i2, i3, i4;

    if ( !paging_mode_log_dirty(d) )
        return;

    /* Shared MFNs should NEVER be marked dirty */
    BUG_ON(dirty && paging_mode_translate(d) && SHARED_M2P(pfn_x(pfn)));

    /*
     * Values with the MSB set denote MFNs that aren't really part of the
     * domain's pseudo-physical memory map (e.g., the shared info frame).
     * Nothing to do here...
     */
    if ( unlikely(!VALID_M2P(pfn_x(pfn))) )
        return;

    BUILD_BUG_ON(paging_logdirty_levels() != 4);
    i1 = L1_LOGDIRTY_IDX(pfn);
    i2 = L2_LOGDIRTY_IDX(pfn);
    i3 = L3_LOGDIRTY_IDX(pfn);
    i4 = L4_LOGDIRTY_IDX(pfn);

    /* Recursive: this is called from inside the shadow code */
    paging_lock_recursive(d);

    if ( unlikely(mfn_eq(d->arch.paging.log_dirty.top, INVALID_MFN)) )
    {
         if ( dirty )
             d->arch.paging.log_dirty.top = paging_new_log_dirty_node(d);
         if ( unlikely(mfn_eq(d->arch.paging.log_dirty.top, INVALID_MFN)) )
             goto out;
    }

    l4 = paging_map_log_dirty_bitmap(d);
    mfn = l4[i4];
    if ( mfn_eq(mfn, INVALID_MFN) && dirty )
        l4[i4] = mfn = paging_new_log_dirty_node(d);
    unmap_domain_page(l4);
    if ( mfn_eq(mfn, INVALID_MFN) )
        goto out;

    l3 = map_domain_page(mfn);
    mfn = l3[i3];
    if ( mfn_eq(mfn, INVALID_MFN) && dirty )
        l3[i3] = mfn = paging_new_log_dirty_node(d);
    unmap_domain_page(l3);
    if ( mfn_eq(mfn, INVALID_MFN) )
        goto out;

    l2 = map_domain_page(mfn);
    mfn = l2[i2];
    if ( mfn_eq(mfn, INVALID_MFN) && dirty )
        l2[i2] = mfn = paging_new_log_dirty_leaf(d);
    unmap_domain_page(l2);
    if ( mfn_eq(mfn, INVALID_MFN) )
        goto out;

    l1 = map_domain_page(mfn);
    changed = dirty ? !__test_and_set_bit(i1, l1)
                    : __test_and_clear_bit(i1, l1);
    unmap_domain_page(l1);
    if ( changed )
    {
        PAGING_DEBUG(LOGDIRTY,
                     "%pd: marked mfn %" PRI_mfn " (pfn %" PRI_pfn ") %s\n",
                     d, mfn_x(mfn), pfn_x(pfn), dirty ? "dirty" : "clean");
        d->arch.paging.log_dirty.dirty_count += dirty ? 1 : -1;
    }

out:
    /* We've already recorded any failed allocations */
    paging_unlock(d);
    return;
}

void paging_mark_pfn_dirty(struct domain *d, pfn_t pfn)
{
    set_pfn_logdirty(d, pfn, true);
}

void paging_mark_pfn_clean(struct domain *d, pfn_t pfn)
{
    set_pfn_logdirty(d, pfn, false);
}

/* Mark a page as dirty */
void paging_mark_dirty(struct domain *d, mfn_t gmfn)
{
    pfn_t pfn;

    if ( !paging_mode_log_dirty(d) || !mfn_valid(gmfn) ||
         page_get_owner(mfn_to_page(gmfn)) != d )
        return;

    /* We /really/ mean PFN here, even for non-translated guests. */
    pfn = _pfn(get_gpfn_from_mfn(mfn_x(gmfn)));

    set_pfn_logdirty(d, pfn, true);
}

#ifdef CONFIG_SHADOW_PAGING
/* Is this guest page dirty? */
bool paging_mfn_is_dirty(const struct domain *d, mfn_t gmfn)
{
    pfn_t pfn;
    mfn_t mfn, *l4, *l3, *l2;
    unsigned long *l1;
    bool dirty;

    ASSERT(paging_locked_by_me(d));
    ASSERT(paging_mode_log_dirty(d));

    /* We /really/ mean PFN here, even for non-translated guests. */
    pfn = _pfn(get_gpfn_from_mfn(mfn_x(gmfn)));
    /* Invalid pages can't be dirty. */
    if ( unlikely(!VALID_M2P(pfn_x(pfn))) )
        return false;

    mfn = d->arch.paging.log_dirty.top;
    if ( mfn_eq(mfn, INVALID_MFN) )
        return false;

    l4 = map_domain_page(mfn);
    mfn = l4[L4_LOGDIRTY_IDX(pfn)];
    unmap_domain_page(l4);
    if ( mfn_eq(mfn, INVALID_MFN) )
        return false;

    l3 = map_domain_page(mfn);
    mfn = l3[L3_LOGDIRTY_IDX(pfn)];
    unmap_domain_page(l3);
    if ( mfn_eq(mfn, INVALID_MFN) )
        return false;

    l2 = map_domain_page(mfn);
    mfn = l2[L2_LOGDIRTY_IDX(pfn)];
    unmap_domain_page(l2);
    if ( mfn_eq(mfn, INVALID_MFN) )
        return false;

    l1 = map_domain_page(mfn);
    dirty = test_bit(L1_LOGDIRTY_IDX(pfn), l1);
    unmap_domain_page(l1);

    return dirty;
}
#endif

/* Read a domain's log-dirty bitmap and stats.  If the operation is a CLEAN,
 * clear the bitmap and stats as well. */
static int paging_log_dirty_op(struct domain *d,
                               struct xen_domctl_shadow_op *sc,
                               bool resuming)
{
    int rv = 0, clean = 0, peek = 1;
    unsigned long pages = 0;
    mfn_t *l4 = NULL, *l3 = NULL, *l2 = NULL;
    unsigned long *l1 = NULL;
    int i4, i3, i2;

    if ( !resuming )
    {
        /*
         * Mark dirty all currently write-mapped pages on e.g. the
         * final iteration of a save operation.
         */
        if ( is_hvm_domain(d) &&
             (sc->mode & XEN_DOMCTL_SHADOW_LOGDIRTY_FINAL) )
            hvm_mapped_guest_frames_mark_dirty(d);

        domain_pause(d);

        /*
         * Flush dirty GFNs potentially cached by hardware. Only need to flush
         * when not resuming, as domain was paused in resuming case therefore
         * it's not possible to have any new dirty pages.
         */
        p2m_flush_hardware_cached_dirty(d);
    }

    paging_lock(d);

    if ( !d->arch.paging.preempt.dom )
        memset(&d->arch.paging.preempt.log_dirty, 0,
               sizeof(d->arch.paging.preempt.log_dirty));
    else if ( d->arch.paging.preempt.dom != current->domain ||
              d->arch.paging.preempt.op != sc->op )
    {
        paging_unlock(d);
        if ( !resuming )
            domain_unpause(d);
        return -EBUSY;
    }

    clean = (sc->op == XEN_DOMCTL_SHADOW_OP_CLEAN);

    PAGING_DEBUG(LOGDIRTY, "log-dirty %s: dom %u faults=%lu dirty=%lu\n",
                 (clean) ? "clean" : "peek",
                 d->domain_id,
                 d->arch.paging.log_dirty.fault_count,
                 d->arch.paging.log_dirty.dirty_count);

    sc->stats.fault_count = min(d->arch.paging.log_dirty.fault_count,
                                UINT32_MAX + 0UL);
    sc->stats.dirty_count = min(d->arch.paging.log_dirty.dirty_count,
                                UINT32_MAX + 0UL);

    if ( guest_handle_is_null(sc->dirty_bitmap) )
        /* caller may have wanted just to clean the state or access stats. */
        peek = 0;

    if ( unlikely(d->arch.paging.log_dirty.failed_allocs) ) {
        printk(XENLOG_WARNING
               "%u failed page allocs while logging dirty pages of d%d\n",
               d->arch.paging.log_dirty.failed_allocs, d->domain_id);
        rv = -ENOMEM;
        goto out;
    }

    l4 = paging_map_log_dirty_bitmap(d);
    i4 = d->arch.paging.preempt.log_dirty.i4;
    i3 = d->arch.paging.preempt.log_dirty.i3;
    pages = d->arch.paging.preempt.log_dirty.done;

    for ( ; (pages < sc->pages) && (i4 < LOGDIRTY_NODE_ENTRIES); i4++, i3 = 0 )
    {
        l3 = ((l4 && !mfn_eq(l4[i4], INVALID_MFN)) ?
              map_domain_page(l4[i4]) : NULL);
        for ( ; (pages < sc->pages) && (i3 < LOGDIRTY_NODE_ENTRIES); i3++ )
        {
            l2 = ((l3 && !mfn_eq(l3[i3], INVALID_MFN)) ?
                  map_domain_page(l3[i3]) : NULL);
            for ( i2 = 0;
                  (pages < sc->pages) && (i2 < LOGDIRTY_NODE_ENTRIES);
                  i2++ )
            {
                unsigned int bytes = PAGE_SIZE;
                l1 = ((l2 && !mfn_eq(l2[i2], INVALID_MFN)) ?
                      map_domain_page(l2[i2]) : NULL);
                if ( unlikely(((sc->pages - pages + 7) >> 3) < bytes) )
                    bytes = (unsigned int)((sc->pages - pages + 7) >> 3);
                if ( likely(peek) )
                {
                    if ( (l1 ? copy_to_guest_offset(sc->dirty_bitmap,
                                                    pages >> 3, (uint8_t *)l1,
                                                    bytes)
                             : clear_guest_offset(sc->dirty_bitmap,
                                                  pages >> 3, bytes)) != 0 )
                    {
                        rv = -EFAULT;
                        goto out;
                    }
                }
                pages += bytes << 3;
                if ( l1 )
                {
                    if ( clean )
                        clear_page(l1);
                    unmap_domain_page(l1);
                }
            }
            if ( l2 )
                unmap_domain_page(l2);

            if ( i3 < LOGDIRTY_NODE_ENTRIES - 1 && hypercall_preempt_check() )
            {
                d->arch.paging.preempt.log_dirty.i4 = i4;
                d->arch.paging.preempt.log_dirty.i3 = i3 + 1;
                rv = -ERESTART;
                break;
            }
        }
        if ( l3 )
            unmap_domain_page(l3);

        if ( !rv && i4 < LOGDIRTY_NODE_ENTRIES - 1 &&
             hypercall_preempt_check() )
        {
            d->arch.paging.preempt.log_dirty.i4 = i4 + 1;
            d->arch.paging.preempt.log_dirty.i3 = 0;
            rv = -ERESTART;
        }
        if ( rv )
            break;
    }
    if ( l4 )
        unmap_domain_page(l4);

    if ( !rv )
    {
        d->arch.paging.preempt.dom = NULL;
        if ( clean )
        {
            d->arch.paging.log_dirty.fault_count = 0;
            d->arch.paging.log_dirty.dirty_count = 0;
        }
    }
    else
    {
        d->arch.paging.preempt.dom = current->domain;
        d->arch.paging.preempt.op = sc->op;
        d->arch.paging.preempt.log_dirty.done = pages;
    }

    paging_unlock(d);

    if ( rv )
    {
        /* Never leave the domain paused on real errors. */
        ASSERT(rv == -ERESTART);
        return rv;
    }

    if ( pages < sc->pages )
        sc->pages = pages;
    if ( clean )
    {
        /* We need to further call clean_dirty_bitmap() functions of specific
         * paging modes (shadow or hap).  Safe because the domain is paused. */
        d->arch.paging.log_dirty.ops->clean(d);
    }
    domain_unpause(d);
    return rv;

 out:
    d->arch.paging.preempt.dom = NULL;
    paging_unlock(d);
    domain_unpause(d);

    if ( l1 )
        unmap_domain_page(l1);
    if ( l2 )
        unmap_domain_page(l2);
    if ( l3 )
        unmap_domain_page(l3);
    if ( l4 )
        unmap_domain_page(l4);

    return rv;
}

/*
 * Callers must supply log_dirty_ops for the log dirty code to call. This
 * function usually is invoked when paging is enabled. Check shadow_enable()
 * and hap_enable() for reference.
 *
 * These function pointers must not be followed with the log-dirty lock held.
 */
void paging_log_dirty_init(struct domain *d, const struct log_dirty_ops *ops)
{
    d->arch.paging.log_dirty.ops = ops;
}

#endif /* PG_log_dirty */

/************************************************/
/*           CODE FOR PAGING SUPPORT            */
/************************************************/
/* Domain paging struct initialization. */
int paging_domain_init(struct domain *d)
{
    int rc;

    if ( (rc = p2m_init(d)) != 0 )
        return rc;

    INIT_PAGE_LIST_HEAD(&d->arch.paging.freelist);
    mm_lock_init(&d->arch.paging.lock);

    /* This must be initialized separately from the rest of the
     * log-dirty init code as that can be called more than once and we
     * don't want to leak any active log-dirty bitmaps */
    d->arch.paging.log_dirty.top = INVALID_MFN;

    /*
     * Shadow pagetables are the default, but we will use
     * hardware assistance if it's available and enabled.
     */
    if ( hap_enabled(d) )
        hap_domain_init(d);
    else
        rc = shadow_domain_init(d);

    return rc;
}

/* vcpu paging struct initialization goes here */
void paging_vcpu_init(struct vcpu *v)
{
    if ( hap_enabled(v->domain) )
        hap_vcpu_init(v);
    else
        shadow_vcpu_init(v);
}

#if PG_log_dirty
int paging_domctl(struct domain *d, struct xen_domctl_shadow_op *sc,
                  XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl,
                  bool resuming)
{
    int rc;

    if ( unlikely(d == current->domain) )
    {
        gdprintk(XENLOG_INFO, "Tried to do a paging op on itself.\n");
        return -EINVAL;
    }

    if ( unlikely(d->is_dying) )
    {
        gdprintk(XENLOG_INFO, "Tried to do a paging op on dying %pd\n", d);
        return -EINVAL;
    }

    if ( unlikely(d->vcpu == NULL) || unlikely(d->vcpu[0] == NULL) )
    {
        gdprintk(XENLOG_DEBUG, "Paging op on a domain (%u) with no vcpus\n",
                 d->domain_id);
        return -EINVAL;
    }

    if ( resuming
         ? (d->arch.paging.preempt.dom != current->domain ||
            d->arch.paging.preempt.op != sc->op)
         : (d->arch.paging.preempt.dom &&
            sc->op != XEN_DOMCTL_SHADOW_OP_GET_ALLOCATION) )
    {
        printk(XENLOG_G_DEBUG
               "%pv: Paging op %#x on Dom%u with unfinished prior op %#x by Dom%u\n",
               current, sc->op, d->domain_id, d->arch.paging.preempt.op,
               d->arch.paging.preempt.dom
               ? d->arch.paging.preempt.dom->domain_id : DOMID_INVALID);
        return -EBUSY;
    }

    /* Code to handle log-dirty. Note that some log dirty operations
     * piggy-back on shadow operations. For example, when
     * XEN_DOMCTL_SHADOW_OP_OFF is called, it first checks whether log dirty
     * mode is enabled. If does, we disables log dirty and continues with
     * shadow code. For this reason, we need to further dispatch domctl
     * to next-level paging code (shadow or hap).
     */
    switch ( sc->op )
    {

    case XEN_DOMCTL_SHADOW_OP_ENABLE:
        if ( !(sc->mode & XEN_DOMCTL_SHADOW_ENABLE_LOG_DIRTY) )
            break;
        fallthrough;
    case XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY:
        return paging_log_dirty_enable(d);

    case XEN_DOMCTL_SHADOW_OP_OFF:
        if ( (rc = paging_log_dirty_disable(d, resuming)) != 0 )
            return rc;
        break;

    case XEN_DOMCTL_SHADOW_OP_CLEAN:
    case XEN_DOMCTL_SHADOW_OP_PEEK:
        if ( sc->mode & ~XEN_DOMCTL_SHADOW_LOGDIRTY_FINAL )
            return -EINVAL;
        return paging_log_dirty_op(d, sc, resuming);
    }

    /* Here, dispatch domctl to the appropriate paging code */
    if ( hap_enabled(d) )
        return hap_domctl(d, sc, u_domctl);
    else
        return shadow_domctl(d, sc, u_domctl);
}

long do_paging_domctl_cont(
    XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
{
    struct xen_domctl op;
    struct domain *d;
    int ret;

    if ( copy_from_guest(&op, u_domctl, 1) )
        return -EFAULT;

    if ( op.interface_version != XEN_DOMCTL_INTERFACE_VERSION ||
         op.cmd != XEN_DOMCTL_shadow_op )
        return -EOPNOTSUPP;

    d = rcu_lock_domain_by_id(op.domain);
    if ( d == NULL )
        return -ESRCH;

    ret = xsm_domctl(XSM_OTHER, d, &op);
    if ( !ret )
    {
        bool lock = !(op.u.shadow_op.op == XEN_DOMCTL_SHADOW_OP_CLEAN ||
                      op.u.shadow_op.op == XEN_DOMCTL_SHADOW_OP_PEEK);

        if ( !lock || domctl_lock_acquire() )
        {
            ret = paging_domctl(d, &op.u.shadow_op, u_domctl, 1);

            if ( lock )
                domctl_lock_release();
        }
        else
            ret = -ERESTART;
    }

    rcu_unlock_domain(d);

    if ( ret == -ERESTART )
        ret = hypercall_create_continuation(__HYPERVISOR_paging_domctl_cont,
                                            "h", u_domctl);
    else if ( __copy_field_to_guest(u_domctl, &op, u.shadow_op) )
        ret = -EFAULT;

    return ret;
}
#endif /* PG_log_dirty */

void paging_vcpu_teardown(struct vcpu *v)
{
    if ( hap_enabled(v->domain) )
        hap_vcpu_teardown(v);
    else
        shadow_vcpu_teardown(v);
}

/* Call when destroying a domain */
int paging_teardown(struct domain *d)
{
    int rc;
    bool preempted = false;

    if ( hap_enabled(d) )
        hap_teardown(d, &preempted);
    else
        shadow_teardown(d, &preempted);

    if ( preempted )
        return -ERESTART;

#if PG_log_dirty
    /* clean up log dirty resources. */
    rc = paging_free_log_dirty_bitmap(d, 0);
    if ( rc == -ERESTART )
        return rc;
#endif

    /* Move populate-on-demand cache back to domain_list for destruction */
    rc = p2m_pod_empty_cache(d);

    return rc;
}

/* Call once all of the references to the domain have gone away */
void paging_final_teardown(struct domain *d)
{
    bool hap = hap_enabled(d);

    PAGING_PRINTK("%pd start: total = %u, free = %u, p2m = %u\n",
                  d, d->arch.paging.total_pages,
                  d->arch.paging.free_pages, d->arch.paging.p2m_pages);

    if ( hap )
        hap_final_teardown(d);

    /*
     * Remove remaining paging memory.  This can be nonzero on certain error
     * paths.
     */
    if ( d->arch.paging.total_pages )
    {
        if ( hap )
            hap_teardown(d, NULL);
        else
            shadow_teardown(d, NULL);
    }

    /* It is now safe to pull down the p2m map. */
    p2m_teardown(p2m_get_hostp2m(d), true, NULL);

    PAGING_PRINTK("%pd done: total = %u, free = %u, p2m = %u\n",
                  d, d->arch.paging.total_pages,
                  d->arch.paging.free_pages, d->arch.paging.p2m_pages);
    ASSERT(!d->arch.paging.p2m_pages);
    ASSERT(!d->arch.paging.free_pages);
    ASSERT(!d->arch.paging.total_pages);

    p2m_final_teardown(d);
}

#ifdef CONFIG_PAGING
/* Enable an arbitrary paging-assistance mode.  Call once at domain
 * creation. */
int paging_enable(struct domain *d, u32 mode)
{
    /* Unrecognised paging mode? */
    if ( mode & ~PG_MASK )
        return -EINVAL;

    /* All of external|translate|refcounts, or none. */
    switch ( mode & (PG_external | PG_translate | PG_refcounts) )
    {
    case 0:
#if PG_external | PG_translate | PG_refcounts
    case PG_external | PG_translate | PG_refcounts:
#endif
        break;
    default:
        return -EINVAL;
    }

    if ( hap_enabled(d) )
        return hap_enable(d, mode);
    else
        return shadow_enable(d, mode);
}
#endif

#ifdef CONFIG_HVM
/* Called from the guest to indicate that a process is being torn down
 * and therefore its pagetables will soon be discarded */
void pagetable_dying(paddr_t gpa)
{
#ifdef CONFIG_SHADOW_PAGING
    struct vcpu *curr = current;

    ASSERT(paging_mode_shadow(curr->domain));

    curr->arch.paging.mode->shadow.pagetable_dying(gpa);
#else
    BUG();
#endif
}
#endif /* CONFIG_HVM */

/* Print paging-assistance info to the console */
void paging_dump_domain_info(struct domain *d)
{
    if ( paging_mode_enabled(d) )
    {
        printk("    paging assistance: ");
        if ( paging_mode_shadow(d) )
            printk("shadow ");
        if ( paging_mode_sh_forced(d) )
            printk("forced ");
        if ( paging_mode_hap(d) )
            printk("hap ");
        if ( paging_mode_refcounts(d) )
            printk("refcounts ");
        if ( paging_mode_log_dirty(d) )
            printk("log_dirty ");
        if ( paging_mode_translate(d) )
            printk("translate ");
        if ( paging_mode_external(d) )
            printk("external ");
        printk("\n");
    }
}

void paging_dump_vcpu_info(struct vcpu *v)
{
    if ( paging_mode_enabled(v->domain) )
    {
        printk("    paging assistance: ");
        if ( paging_mode_shadow(v->domain) )
            printk("shadowed %u-on-%u\n",
                   paging_get_hostmode(v)->guest_levels,
                   paging_get_hostmode(v)->shadow.shadow_levels);
        else
            printk("hap, %u levels\n",
                   paging_get_hostmode(v)->guest_levels);
    }
}

#ifdef CONFIG_HVM
void paging_update_nestedmode(struct vcpu *v)
{
    ASSERT(nestedhvm_enabled(v->domain));
    if (nestedhvm_paging_mode_hap(v))
        /* nested-on-nested */
        v->arch.paging.nestedmode = hap_paging_get_mode(v);
    else
        /* TODO: shadow-on-shadow */
        v->arch.paging.nestedmode = NULL;
    hvm_asid_flush_vcpu(v);
}

int __init paging_set_allocation(struct domain *d, unsigned int pages,
                                 bool *preempted)
{
    int rc;

    ASSERT(paging_mode_enabled(d));

    paging_lock(d);
    if ( hap_enabled(d) )
        rc = hap_set_allocation(d, pages, preempted);
    else
        rc = shadow_set_allocation(d, pages, preempted);
    paging_unlock(d);

    return rc;
}
#endif

int arch_get_paging_mempool_size(struct domain *d, uint64_t *size)
{
    unsigned long pages;

    if ( is_pv_domain(d) )                 /* TODO: Relax in due course */
        return -EOPNOTSUPP;

    pages  = d->arch.paging.total_pages;
    pages += d->arch.paging.p2m_pages;

    *size = pages << PAGE_SHIFT;

    return 0;
}

int arch_set_paging_mempool_size(struct domain *d, uint64_t size)
{
    unsigned long pages = size >> PAGE_SHIFT;
    bool preempted = false;
    int rc;

    if ( is_pv_domain(d) )                 /* TODO: Relax in due course */
        return -EOPNOTSUPP;

    if ( size & ~PAGE_MASK ||              /* Non page-sized request? */
         pages != (unsigned int)pages )    /* Overflow $X_set_allocation()? */
        return -EINVAL;

    paging_lock(d);
    if ( hap_enabled(d) )
        rc = hap_set_allocation(d, pages, &preempted);
    else
        rc = shadow_set_allocation(d, pages, &preempted);
    paging_unlock(d);

    /*
     * TODO: Adjust $X_set_allocation() so this is true.
    ASSERT(preempted == (rc == -ERESTART));
     */

    return preempted ? -ERESTART : rc;
}

/*
 * Local variables:
 * mode: C
 * c-file-style: "BSD"
 * c-basic-offset: 4
 * indent-tabs-mode: nil
 * End:
 */
