/******************************************************************************
 * domain_page.h
 *
 * Allow temporary mapping of domain pages.
 *
 * Copyright (c) 2003-2006, Keir Fraser <keir@xensource.com>
 */

#include <xen/domain_page.h>
#include <xen/efi.h>
#include <xen/mm.h>
#include <xen/perfc.h>
#include <xen/pfn.h>
#include <xen/sched.h>
#include <xen/vmap.h>
#include <asm/current.h>
#include <asm/flushtlb.h>
#include <asm/hardirq.h>
#include <asm/setup.h>

static inline struct vcpu *mapcache_current_vcpu(void)
{
    struct vcpu *v = this_cpu(pgtable_vcpu);
    struct vcpu *curr = current;

    /*
     * During early boot pgtable_vcpu is not set, callers must handle NULL.
     * Non-PV domains don't have a mapcache, the directmap covers all physical
     * address space.
     */
    if ( !v || !is_pv_vcpu(v) )
        return NULL;

    /*
     * If we are in a lazy context-switch state from a PV vCPU do a full switch
     * to the idle vCPU now, otherwise an incoming FLUSH_VCPU_STATE IPI would
     * change the page tables under our feet an invalidate any in-use mapcache
     * entries.
     */
    if ( unlikely(this_cpu(curr_vcpu) != curr) )
    {
        ASSERT(curr == idle_vcpu[smp_processor_id()]);
        sync_local_execstate();
        /* We must now be running on the idle page table. */
        ASSERT(cr3_pa(read_cr3()) == __pa(idle_pg_table));
    }

    /*
     * At this point we can guarantee Xen is not in lazy context switch: either
     * the code above will have synced the state, or an incoming
     * FLUSH_VCPU_STATE IPI has done so behind our back.  Use ACCESS_ONCE to
     * ensure the compiler never returns the locally cached pgtable_vcpu value.
     */
    return ACCESS_ONCE(this_cpu(pgtable_vcpu));
}

#define mapcache_l2_entry(e) ((e) >> PAGETABLE_ORDER)
#define MAPCACHE_L2_ENTRIES (mapcache_l2_entry(MAPCACHE_ENTRIES - 1) + 1)
#define MAPCACHE_L1ENT(idx) \
    __linear_l1_table[l1_linear_offset(MAPCACHE_VIRT_START + pfn_to_paddr(idx))]

void *map_domain_page(mfn_t mfn)
{
    unsigned long flags;
    unsigned int idx, i;
    struct vcpu *v;
    struct mapcache_domain *dcache;
    struct mapcache_vcpu *vcache;
    struct vcpu_maphash_entry *hashent;

#ifdef NDEBUG
    if ( mfn_x(mfn) <= PFN_DOWN(__pa(HYPERVISOR_VIRT_END - 1)) )
        return mfn_to_virt(mfn_x(mfn));
#endif

    v = mapcache_current_vcpu();
    if ( !v || !is_pv_vcpu(v) )
        return mfn_to_virt(mfn_x(mfn));

    dcache = &v->domain->arch.pv.mapcache;
    vcache = &v->arch.pv.mapcache;
    if ( !dcache->inuse )
        return mfn_to_virt(mfn_x(mfn));

    perfc_incr(map_domain_page_count);

    local_irq_save(flags);

    hashent = &vcache->hash[MAPHASH_HASHFN(mfn_x(mfn))];
    if ( hashent->mfn == mfn_x(mfn) )
    {
        idx = hashent->idx;
        ASSERT(idx < dcache->entries);
        hashent->refcnt++;
        ASSERT(hashent->refcnt);
        ASSERT(mfn_eq(l1e_get_mfn(MAPCACHE_L1ENT(idx)), mfn));
        goto out;
    }

    spin_lock(&dcache->lock);

    /* Has some other CPU caused a wrap? We must flush if so. */
    if ( unlikely(dcache->epoch != vcache->shadow_epoch) )
    {
        vcache->shadow_epoch = dcache->epoch;
        if ( NEED_FLUSH(this_cpu(tlbflush_time), dcache->tlbflush_timestamp) )
        {
            perfc_incr(domain_page_tlb_flush);
            flush_tlb_local();
        }
    }

    idx = find_next_zero_bit(dcache->inuse, dcache->entries, dcache->cursor);
    if ( unlikely(idx >= dcache->entries) )
    {
        unsigned long accum = 0, prev = 0;

        /* /First/, clean the garbage map and update the inuse list. */
        for ( i = 0; i < BITS_TO_LONGS(dcache->entries); i++ )
        {
            accum |= prev;
            dcache->inuse[i] &= ~xchg(&dcache->garbage[i], 0);
            prev = ~dcache->inuse[i];
        }

        if ( accum | (prev & BITMAP_LAST_WORD_MASK(dcache->entries)) )
            idx = find_first_zero_bit(dcache->inuse, dcache->entries);
        else
        {
            /* Replace a hash entry instead. */
            i = MAPHASH_HASHFN(mfn_x(mfn));
            do {
                hashent = &vcache->hash[i];
                if ( hashent->idx != MAPHASHENT_NOTINUSE && !hashent->refcnt )
                {
                    idx = hashent->idx;
                    ASSERT(l1e_get_pfn(MAPCACHE_L1ENT(idx)) == hashent->mfn);
                    l1e_write(&MAPCACHE_L1ENT(idx), l1e_empty());
                    hashent->idx = MAPHASHENT_NOTINUSE;
                    hashent->mfn = ~0UL;
                    break;
                }
                if ( ++i == MAPHASH_ENTRIES )
                    i = 0;
            } while ( i != MAPHASH_HASHFN(mfn_x(mfn)) );
        }
        BUG_ON(idx >= dcache->entries);

        /* /Second/, flush TLBs. */
        perfc_incr(domain_page_tlb_flush);
        flush_tlb_local();
        vcache->shadow_epoch = ++dcache->epoch;
        dcache->tlbflush_timestamp = tlbflush_current_time();
    }

    set_bit(idx, dcache->inuse);
    dcache->cursor = idx + 1;

    spin_unlock(&dcache->lock);

    l1e_write(&MAPCACHE_L1ENT(idx), l1e_from_mfn(mfn, __PAGE_HYPERVISOR_RW));

 out:
    local_irq_restore(flags);
    return (void *)MAPCACHE_VIRT_START + pfn_to_paddr(idx);
}

void unmap_domain_page(const void *ptr)
{
    unsigned int idx;
    struct vcpu *v;
    struct mapcache_domain *dcache;
    unsigned long va = (unsigned long)ptr, mfn, flags;
    struct vcpu_maphash_entry *hashent;

    if ( !va || va >= DIRECTMAP_VIRT_START )
        return;

    ASSERT(va >= MAPCACHE_VIRT_START && va < MAPCACHE_VIRT_END);

    v = mapcache_current_vcpu();
    ASSERT(v && is_pv_vcpu(v));

    dcache = &v->domain->arch.pv.mapcache;
    ASSERT(dcache->inuse);

    idx = PFN_DOWN(va - MAPCACHE_VIRT_START);
    mfn = l1e_get_pfn(MAPCACHE_L1ENT(idx));
    hashent = &v->arch.pv.mapcache.hash[MAPHASH_HASHFN(mfn)];

    local_irq_save(flags);

    if ( hashent->idx == idx )
    {
        ASSERT(hashent->mfn == mfn);
        ASSERT(hashent->refcnt);
        hashent->refcnt--;
    }
    else if ( !hashent->refcnt )
    {
        if ( hashent->idx != MAPHASHENT_NOTINUSE )
        {
            /* /First/, zap the PTE. */
            ASSERT(l1e_get_pfn(MAPCACHE_L1ENT(hashent->idx)) ==
                   hashent->mfn);
            l1e_write(&MAPCACHE_L1ENT(hashent->idx), l1e_empty());
            /* /Second/, mark as garbage. */
            set_bit(hashent->idx, dcache->garbage);
        }

        /* Add newly-freed mapping to the maphash. */
        hashent->mfn = mfn;
        hashent->idx = idx;
    }
    else
    {
        /* /First/, zap the PTE. */
        l1e_write(&MAPCACHE_L1ENT(idx), l1e_empty());
        /* /Second/, mark as garbage. */
        set_bit(idx, dcache->garbage);
    }

    local_irq_restore(flags);
}

int mapcache_domain_init(struct domain *d)
{
    struct mapcache_domain *dcache = &d->arch.pv.mapcache;
    unsigned int bitmap_pages;

    ASSERT(is_pv_domain(d));

#ifdef NDEBUG
    if ( !mem_hotplug && max_page <= PFN_DOWN(__pa(HYPERVISOR_VIRT_END - 1)) )
        return 0;
#endif

    BUILD_BUG_ON(MAPCACHE_VIRT_END + PAGE_SIZE * (3 +
                 2 * PFN_UP(BITS_TO_LONGS(MAPCACHE_ENTRIES) * sizeof(long))) >
                 MAPCACHE_VIRT_START + (PERDOMAIN_SLOT_MBYTES << 20));
    bitmap_pages = PFN_UP(BITS_TO_LONGS(MAPCACHE_ENTRIES) * sizeof(long));
    dcache->inuse = (void *)MAPCACHE_VIRT_END + PAGE_SIZE;
    dcache->garbage = dcache->inuse +
                      (bitmap_pages + 1) * PAGE_SIZE / sizeof(long);

    spin_lock_init(&dcache->lock);

    return create_perdomain_mapping(d, (unsigned long)dcache->inuse,
                                    2 * bitmap_pages + 1,
                                    NIL(l1_pgentry_t *), NULL);
}

int mapcache_vcpu_init(struct vcpu *v)
{
    struct domain *d = v->domain;
    struct mapcache_domain *dcache = &d->arch.pv.mapcache;
    unsigned long i;
    unsigned int ents = d->max_vcpus * MAPCACHE_VCPU_ENTRIES;
    unsigned int nr = PFN_UP(BITS_TO_LONGS(ents) * sizeof(long));

    if ( !is_pv_vcpu(v) || !dcache->inuse )
        return 0;

    if ( ents > dcache->entries )
    {
        /* Populate page tables. */
        int rc = create_perdomain_mapping(d, MAPCACHE_VIRT_START, ents,
                                          NIL(l1_pgentry_t *), NULL);

        /* Populate bit maps. */
        if ( !rc )
            rc = create_perdomain_mapping(d, (unsigned long)dcache->inuse,
                                          nr, NULL, NIL(struct page_info *));
        if ( !rc )
            rc = create_perdomain_mapping(d, (unsigned long)dcache->garbage,
                                          nr, NULL, NIL(struct page_info *));

        if ( rc )
            return rc;

        dcache->entries = ents;
    }

    /* Mark all maphash entries as not in use. */
    BUILD_BUG_ON(MAPHASHENT_NOTINUSE < MAPCACHE_ENTRIES);
    for ( i = 0; i < MAPHASH_ENTRIES; i++ )
    {
        struct vcpu_maphash_entry *hashent = &v->arch.pv.mapcache.hash[i];

        hashent->mfn = ~0UL; /* never valid to map */
        hashent->idx = MAPHASHENT_NOTINUSE;
    }

    return 0;
}

void *map_domain_page_global(mfn_t mfn)
{
    ASSERT(!in_irq() &&
           ((system_state >= SYS_STATE_boot &&
             system_state < SYS_STATE_active) ||
            local_irq_is_enabled()));

#ifdef NDEBUG
    if ( mfn_x(mfn) <= PFN_DOWN(__pa(HYPERVISOR_VIRT_END - 1)) )
        return mfn_to_virt(mfn_x(mfn));
#endif

    return vmap(&mfn, 1);
}

void unmap_domain_page_global(const void *ptr)
{
    unsigned long va = (unsigned long)ptr;

    if ( va >= DIRECTMAP_VIRT_START )
        return;

    ASSERT(va >= VMAP_VIRT_START && va < VMAP_VIRT_END);

    vunmap(ptr);
}

/* Translate a map-domain-page'd address to the underlying MFN */
mfn_t domain_page_map_to_mfn(const void *ptr)
{
    unsigned long va = (unsigned long)ptr;

    if ( va >= DIRECTMAP_VIRT_START )
        return _mfn(virt_to_mfn(ptr));

    if ( va >= VMAP_VIRT_START && va < VMAP_VIRT_END )
        return vmap_to_mfn(va);

    ASSERT(va >= MAPCACHE_VIRT_START && va < MAPCACHE_VIRT_END);

    return l1e_get_mfn(__linear_l1_table[l1_linear_offset(va)]);
}
