/* SPDX-License-Identifier: GPL-2.0-only */
#include <xen/cpu.h>
#include <xen/cpumask.h>
#include <xen/init.h>
#include <xen/mm.h>
#include <xen/param.h>
#include <xen/sizes.h>
#include <xen/smp.h>
#include <xen/spinlock.h>
#include <xen/vmap.h>
#include <xen/warning.h>
#include <xen/notifier.h>
#include <asm/cpufeature.h>
#include <asm/cpuerrata.h>
#include <asm/insn.h>
#include <asm/psci.h>

/* Override macros from asm/page.h to make them work with mfn_t */
#undef virt_to_mfn
#define virt_to_mfn(va) _mfn(__virt_to_mfn(va))

/* Hardening Branch predictor code for Arm64 */
#ifdef CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR

#define VECTOR_TABLE_SIZE SZ_2K

/*
 * Number of available table vectors (this should be in-sync with
 * arch/arm64/bpi.S
 */
#define NR_BPI_HYP_VECS 4

extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[];

/*
 * Key for each slot. This is used to find whether a specific workaround
 * had a slot assigned.
 *
 * The key is virtual address of the vector workaround
 */
static uintptr_t bp_harden_slot_key[NR_BPI_HYP_VECS];

/*
 * [hyp_vec_start, hyp_vec_end[ corresponds to the first 31 instructions
 * of each vector. The last (i.e 32th) instruction is used to branch to
 * the original entry.
 *
 * Those instructions will be copied on each vector to harden them.
 */
static bool copy_hyp_vect_bpi(unsigned int slot, const char *hyp_vec_start,
                              const char *hyp_vec_end)
{
    void *dst_remapped;
    const void *dst = __bp_harden_hyp_vecs_start + slot * VECTOR_TABLE_SIZE;
    unsigned int i;
    mfn_t dst_mfn = virt_to_mfn(dst);

    BUG_ON(((hyp_vec_end - hyp_vec_start) / 4) > 31);

    /*
     * Vectors are part of the text that are mapped read-only. So re-map
     * the vector table to be able to update vectors.
     */
    dst_remapped = vmap_contig(dst_mfn,
                               1UL << get_order_from_bytes(VECTOR_TABLE_SIZE));
    if ( !dst_remapped )
        return false;

    dst_remapped += (vaddr_t)dst & ~PAGE_MASK;

    for ( i = 0; i < VECTOR_TABLE_SIZE; i += 0x80 )
    {
        memcpy(dst_remapped + i, hyp_vec_start, hyp_vec_end - hyp_vec_start);
    }

    clean_dcache_va_range(dst_remapped, VECTOR_TABLE_SIZE);
    invalidate_icache();

    vunmap((void *)((vaddr_t)dst_remapped & PAGE_MASK));

    return true;
}

static bool __maybe_unused
install_bp_hardening_vec(const struct arm_cpu_capabilities *entry,
                         const char *hyp_vec_start,
                         const char *hyp_vec_end,
                         const char *desc)
{
    static int last_slot = -1;
    static DEFINE_SPINLOCK(bp_lock);
    unsigned int i, slot = -1;
    bool ret = true;

    /*
     * Enable callbacks are called on every CPU based on the
     * capabilities. So double-check whether the CPU matches the
     * entry.
     */
    if ( !entry->matches(entry) )
        return true;

    printk(XENLOG_INFO "CPU%u will %s on exception entry\n",
           smp_processor_id(), desc);

    spin_lock(&bp_lock);

    /*
     * Look up whether the hardening vector had a slot already
     * assigned.
     */
    for ( i = 0; i < 4; i++ )
    {
        if ( bp_harden_slot_key[i] == (uintptr_t)hyp_vec_start )
        {
            slot = i;
            break;
        }
    }

    if ( slot == -1 )
    {
        last_slot++;
        /* Check we don't overrun the number of slots available. */
        BUG_ON(NR_BPI_HYP_VECS <= last_slot);

        slot = last_slot;
        ret = copy_hyp_vect_bpi(slot, hyp_vec_start, hyp_vec_end);

        /* Only update the slot if the copy succeeded. */
        if ( ret )
            bp_harden_slot_key[slot] = (uintptr_t)hyp_vec_start;
    }

    if ( ret )
    {
        /* Install the new vector table. */
        WRITE_SYSREG((vaddr_t)(__bp_harden_hyp_vecs_start + slot * VECTOR_TABLE_SIZE),
                     VBAR_EL2);
        isb();
    }

    spin_unlock(&bp_lock);

    return ret;
}

extern char __smccc_workaround_smc_start_1[], __smccc_workaround_smc_end_1[];
extern char __smccc_workaround_smc_start_3[], __smccc_workaround_smc_end_3[];
extern char __mitigate_spectre_bhb_clear_insn_start[],
            __mitigate_spectre_bhb_clear_insn_end[];
extern char __mitigate_spectre_bhb_loop_start_8[],
            __mitigate_spectre_bhb_loop_end_8[];
extern char __mitigate_spectre_bhb_loop_start_24[],
            __mitigate_spectre_bhb_loop_end_24[];
extern char __mitigate_spectre_bhb_loop_start_32[],
            __mitigate_spectre_bhb_loop_end_32[];

static int enable_smccc_arch_workaround_1(void *data)
{
    struct arm_smccc_res res;
    const struct arm_cpu_capabilities *entry = data;

    /*
     * Enable callbacks are called on every CPU based on the
     * capabilities. So double-check whether the CPU matches the
     * entry.
     */
    if ( !entry->matches(entry) )
        return 0;

    /*
     * No need to install hardened vector when the processor has
     * ID_AA64PRF0_EL1.CSV2 set.
     */
    if ( cpu_data[smp_processor_id()].pfr64.csv2 )
        return 0;

    if ( smccc_ver < SMCCC_VERSION(1, 1) )
        goto warn;

    arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FID,
                      ARM_SMCCC_ARCH_WORKAROUND_1_FID, &res);
    /* The return value is in the lower 32-bits. */
    if ( (int)res.a0 < 0 )
        goto warn;

    return !install_bp_hardening_vec(entry,__smccc_workaround_smc_start_1,
                                     __smccc_workaround_smc_end_1,
                                     "call ARM_SMCCC_ARCH_WORKAROUND_1");

warn:
    printk_once("**** No support for ARM_SMCCC_ARCH_WORKAROUND_1. ****\n"
                "**** Please update your firmware.                ****\n");

    return 0;
}

/*
 * Spectre BHB Mitigation
 *
 * CPU is either:
 * - Having CVS2.3 so it is not affected.
 * - Having ECBHB and is clearing the branch history buffer when an exception
 *   to a different exception level is happening so no mitigation is needed.
 * - Mitigating using a loop on exception entry (number of loop depending on
 *   the CPU).
 * - Mitigating using the firmware.
 */
static int enable_spectre_bhb_workaround(void *data)
{
    const struct arm_cpu_capabilities *entry = data;

    /*
     * Enable callbacks are called on every CPU based on the capabilities, so
     * double-check whether the CPU matches the entry.
     */
    if ( !entry->matches(entry) )
        return 0;

    if ( cpu_data[smp_processor_id()].pfr64.csv2 == 3 )
        return 0;

    if ( cpu_data[smp_processor_id()].mm64.ecbhb )
        return 0;

    if ( cpu_data[smp_processor_id()].isa64.clearbhb )
        return !install_bp_hardening_vec(entry,
                                    __mitigate_spectre_bhb_clear_insn_start,
                                    __mitigate_spectre_bhb_clear_insn_end,
                                     "use clearBHB instruction");

    /* Apply solution depending on hwcaps set on arm_errata */
    if ( cpus_have_cap(ARM_WORKAROUND_BHB_LOOP_8) )
        return !install_bp_hardening_vec(entry,
                                         __mitigate_spectre_bhb_loop_start_8,
                                         __mitigate_spectre_bhb_loop_end_8,
                                         "use 8 loops workaround");

    if ( cpus_have_cap(ARM_WORKAROUND_BHB_LOOP_24) )
        return !install_bp_hardening_vec(entry,
                                         __mitigate_spectre_bhb_loop_start_24,
                                         __mitigate_spectre_bhb_loop_end_24,
                                         "use 24 loops workaround");

    if ( cpus_have_cap(ARM_WORKAROUND_BHB_LOOP_32) )
        return !install_bp_hardening_vec(entry,
                                         __mitigate_spectre_bhb_loop_start_32,
                                         __mitigate_spectre_bhb_loop_end_32,
                                         "use 32 loops workaround");

    if ( cpus_have_cap(ARM_WORKAROUND_BHB_SMCC_3) )
    {
        struct arm_smccc_res res;

        if ( smccc_ver < SMCCC_VERSION(1, 1) )
            goto warn;

        arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FID,
                          ARM_SMCCC_ARCH_WORKAROUND_3_FID, &res);
        /* The return value is in the lower 32-bits. */
        if ( (int)res.a0 < 0 )
        {
            /*
             * On processor affected with CSV2=0, workaround 1 will mitigate
             * both Spectre v2 and BHB so use it when available
             */
            if ( enable_smccc_arch_workaround_1(data) )
                return 1;

            goto warn;
        }

        return !install_bp_hardening_vec(entry,__smccc_workaround_smc_start_3,
                                         __smccc_workaround_smc_end_3,
                                         "call ARM_SMCCC_ARCH_WORKAROUND_3");
    }

warn:
    printk_once("**** No support for any spectre BHB workaround.  ****\n"
                "**** Please update your firmware.                ****\n");

    return 0;
}

#endif /* CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR */

/* Hardening Branch predictor code for Arm32 */
#ifdef CONFIG_ARM32_HARDEN_BRANCH_PREDICTOR

/*
 * Per-CPU vector tables to use when returning to the guests. They will
 * only be used on platform requiring to harden the branch predictor.
 */
DEFINE_PER_CPU_READ_MOSTLY(const char *, bp_harden_vecs);

extern char hyp_traps_vector_bp_inv[];
extern char hyp_traps_vector_ic_inv[];

static void __maybe_unused
install_bp_hardening_vecs(const struct arm_cpu_capabilities *entry,
                          const char *hyp_vecs, const char *desc)
{
    /*
     * Enable callbacks are called on every CPU based on the
     * capabilities. So double-check whether the CPU matches the
     * entry.
     */
    if ( !entry->matches(entry) )
        return;

    printk(XENLOG_INFO "CPU%u will %s on guest exit\n",
           smp_processor_id(), desc);
    this_cpu(bp_harden_vecs) = hyp_vecs;
}

static int enable_bp_inv_hardening(void *data)
{
    install_bp_hardening_vecs(data, hyp_traps_vector_bp_inv,
                              "execute BPIALL");
    return 0;
}

static int enable_ic_inv_hardening(void *data)
{
    install_bp_hardening_vecs(data, hyp_traps_vector_ic_inv,
                              "execute ICIALLU");
    return 0;
}

#endif

#ifdef CONFIG_ARM_SSBD

enum ssbd_state ssbd_state = ARM_SSBD_RUNTIME;

static int __init parse_spec_ctrl(const char *s)
{
    const char *ss;
    int rc = 0;

    do {
        ss = strchr(s, ',');
        if ( !ss )
            ss = strchr(s, '\0');

        if ( !strncmp(s, "ssbd=", 5) )
        {
            s += 5;

            if ( !cmdline_strcmp(s, "force-disable") )
                ssbd_state = ARM_SSBD_FORCE_DISABLE;
            else if ( !cmdline_strcmp(s, "runtime") )
                ssbd_state = ARM_SSBD_RUNTIME;
            else if ( !cmdline_strcmp(s, "force-enable") )
                ssbd_state = ARM_SSBD_FORCE_ENABLE;
            else
                rc = -EINVAL;
        }
        else
            rc = -EINVAL;

        s = ss + 1;
    } while ( *ss );

    return rc;
}
custom_param("spec-ctrl", parse_spec_ctrl);

/* Arm64 only for now as for Arm32 the workaround is currently handled in C. */
#ifdef CONFIG_ARM_64
void asmlinkage __init arm_enable_wa2_handling(const struct alt_instr *alt,
                                               const uint32_t *origptr,
                                               uint32_t *updptr, int nr_inst)
{
    BUG_ON(nr_inst != 1);

    /*
     * Only allow mitigation on guest ARCH_WORKAROUND_2 if the SSBD
     * state allow it to be flipped.
     */
    if ( get_ssbd_state() == ARM_SSBD_RUNTIME )
        *updptr = aarch64_insn_gen_nop();
}
#endif

/*
 * Assembly code may use the variable directly, so we need to make sure
 * it fits in a register.
 */
DEFINE_PER_CPU_READ_MOSTLY(register_t, ssbd_callback_required);

static bool has_ssbd_mitigation(const struct arm_cpu_capabilities *entry)
{
    struct arm_smccc_res res;
    bool required;

    if ( smccc_ver < SMCCC_VERSION(1, 1) )
        return false;

    arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FID,
                      ARM_SMCCC_ARCH_WORKAROUND_2_FID, &res);

    switch ( (int)res.a0 )
    {
    case ARM_SMCCC_NOT_SUPPORTED:
        ssbd_state = ARM_SSBD_UNKNOWN;
        return false;

    case ARM_SMCCC_NOT_REQUIRED:
        ssbd_state = ARM_SSBD_MITIGATED;
        return false;

    case ARM_SMCCC_SUCCESS:
        required = true;
        break;

    case 1: /* Mitigation not required on this CPU. */
        required = false;
        break;

    default:
        ASSERT_UNREACHABLE();
        return false;
    }

    switch ( ssbd_state )
    {
    case ARM_SSBD_FORCE_DISABLE:
        printk_once("%s disabled from command-line\n", entry->desc);

        arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2_FID, 0, NULL);
        required = false;
        break;

    case ARM_SSBD_RUNTIME:
        if ( required )
        {
            this_cpu(ssbd_callback_required) = 1;
            arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2_FID, 1, NULL);
        }

        break;

    case ARM_SSBD_FORCE_ENABLE:
        printk_once("%s forced from command-line\n", entry->desc);

        arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2_FID, 1, NULL);
        required = true;
        break;

    default:
        ASSERT_UNREACHABLE();
        return false;
    }

    return required;
}
#endif

#define MIDR_RANGE(model, min, max)     \
    .matches = is_affected_midr_range,  \
    .midr_model = (model),              \
    .midr_range_min = (min),            \
    .midr_range_max = (max)

#define MIDR_ALL_VERSIONS(model)        \
    .matches = is_affected_midr_range,  \
    .midr_model = (model),              \
    .midr_range_min = 0,                \
    .midr_range_max = (MIDR_VARIANT_MASK | MIDR_REVISION_MASK)

static bool __maybe_unused
is_affected_midr_range(const struct arm_cpu_capabilities *entry)
{
    return MIDR_IS_CPU_MODEL_RANGE(current_cpu_data.midr.bits, entry->midr_model,
                                   entry->midr_range_min,
                                   entry->midr_range_max);
}

static const struct arm_cpu_capabilities arm_errata[] = {
    {
        /* Cortex-A15 r0p4 */
        .desc = "ARM erratum 766422",
        .capability = ARM32_WORKAROUND_766422,
        MIDR_RANGE(MIDR_CORTEX_A15, 0x04, 0x04),
    },
#if defined(CONFIG_ARM64_ERRATUM_827319) || \
    defined(CONFIG_ARM64_ERRATUM_824069)
    {
        /* Cortex-A53 r0p[012] */
        .desc = "ARM errata 827319, 824069",
        .capability = ARM64_WORKAROUND_CLEAN_CACHE,
        MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02),
    },
#endif
#ifdef CONFIG_ARM64_ERRATUM_819472
    {
        /* Cortex-A53 r0[01] */
        .desc = "ARM erratum 819472",
        .capability = ARM64_WORKAROUND_CLEAN_CACHE,
        MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01),
    },
#endif
#ifdef CONFIG_ARM64_ERRATUM_832075
    {
        /* Cortex-A57 r0p0 - r1p2 */
        .desc = "ARM erratum 832075",
        .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
        MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
                   (1 << MIDR_VARIANT_SHIFT) | 2),
    },
#endif
#ifdef CONFIG_ARM64_ERRATUM_834220
    {
        /* Cortex-A57 r0p0 - r1p2 */
        .desc = "ARM erratum 834220",
        .capability = ARM64_WORKAROUND_834220,
        MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
                   (1 << MIDR_VARIANT_SHIFT) | 2),
    },
#endif
#ifdef CONFIG_ARM64_ERRATUM_1286807
    {
        /* Cortex-A76 r0p0 - r3p0 */
        .desc = "ARM erratum 1286807",
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_RANGE(MIDR_CORTEX_A76, 0, 3 << MIDR_VARIANT_SHIFT),
    },
    {
        /* Neoverse-N1 r0p0 - r3p0 */
        .desc = "ARM erratum 1286807",
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 3 << MIDR_VARIANT_SHIFT),
    },
#endif
#ifdef CONFIG_ARM64_ERRATUM_CVE_2025_10263
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X4),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X925),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_C1_ULTRA),
    },
    {
        .capability = ARM64_WORKAROUND_REPEAT_TLBI,
        MIDR_ALL_VERSIONS(MIDR_C1_PREMIUM),
    },
#endif
#ifdef CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR
    {
        .capability = ARM_HARDEN_BRANCH_PREDICTOR,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
        .enable = enable_smccc_arch_workaround_1,
    },
    {
        .capability = ARM_HARDEN_BRANCH_PREDICTOR,
        MIDR_RANGE(MIDR_CORTEX_A72, 0, 1 << MIDR_VARIANT_SHIFT),
        .enable = enable_smccc_arch_workaround_1,
    },
    {
        .capability = ARM_WORKAROUND_BHB_SMCC_3,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_SMCC_3,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
        .enable = enable_spectre_bhb_workaround,
    },
    /* spectre BHB */
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_8,
        MIDR_RANGE(MIDR_CORTEX_A72, 1 << MIDR_VARIANT_SHIFT,
                   (MIDR_VARIANT_MASK | MIDR_REVISION_MASK)),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_24,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_24,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_24,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
        .enable = enable_spectre_bhb_workaround,
    },
    {
        .capability = ARM_WORKAROUND_BHB_LOOP_32,
        MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
        .enable = enable_spectre_bhb_workaround,
    },

#endif
#ifdef CONFIG_ARM32_HARDEN_BRANCH_PREDICTOR
    {
        .capability = ARM_HARDEN_BRANCH_PREDICTOR,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A12),
        .enable = enable_bp_inv_hardening,
    },
    {
        .capability = ARM_HARDEN_BRANCH_PREDICTOR,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A17),
        .enable = enable_bp_inv_hardening,
    },
    {
        .capability = ARM_HARDEN_BRANCH_PREDICTOR,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A15),
        .enable = enable_ic_inv_hardening,
    },
#endif
#ifdef CONFIG_ARM_SSBD
    {
        .desc = "Speculative Store Bypass Disabled",
        .capability = ARM_SSBD,
        .matches = has_ssbd_mitigation,
    },
#endif
#ifdef CONFIG_ARM_ERRATUM_858921
    {
        /* Cortex-A73 (all versions) */
        .desc = "ARM erratum 858921",
        .capability = ARM_WORKAROUND_858921,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
    },
#endif
    {
        /* Neoverse r0p0 - r2p0 */
        .desc = "ARM erratum 1165522",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 2 << MIDR_VARIANT_SHIFT),
    },
    {
        /* Cortex-A76 r0p0 - r2p0 */
        .desc = "ARM erratum 1165522",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_RANGE(MIDR_CORTEX_A76, 0, 2 << MIDR_VARIANT_SHIFT),
    },
    {
        .desc = "ARM erratum 1319537",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
    },
    {
        .desc = "ARM erratum 1319367",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
    },
#ifdef CONFIG_ARM64_ERRATUM_1508412
    {
        /* Cortex-A77 r0p0 - r1p0 */
        .desc = "ARM erratum 1508412 (hypervisor portion)",
        .capability = ARM64_WORKAROUND_1508412,
        MIDR_RANGE(MIDR_CORTEX_A77, 0, 1),
    },
#endif
    {
        /* Cortex-A55 (All versions as erratum is open in SDEN v14) */
        .desc = "ARM erratum 1530923",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
    },
    {
        /* Cortex-A53 (All versions) */
        .desc = "ARM erratum 1530924",
        .capability = ARM64_WORKAROUND_AT_SPECULATE,
        MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
    },
    {},
};

void check_local_cpu_errata(void)
{
    update_cpu_capabilities(arm_errata, "enabled workaround for");
}

void __init enable_errata_workarounds(void)
{
    enable_cpu_capabilities(arm_errata);

#if defined(CONFIG_ARM64_ERRATUM_832075) || defined(CONFIG_ARM64_ERRATUM_1508412)
    if ( cpus_have_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
         cpus_have_cap(ARM64_WORKAROUND_1508412) )
    {
        printk_once("**** Guests without CPU erratum workarounds can deadlock the system! ****\n"
                    "**** Only trusted guests should be used.                             ****\n");

        /* Taint the machine has being insecure */
        add_taint(TAINT_MACHINE_INSECURE);
    }
#endif
}

static int cpu_errata_callback(struct notifier_block *nfb,
                               unsigned long action,
                               void *hcpu)
{
    int rc = 0;

    switch ( action )
    {
    case CPU_STARTING:
        /*
         * At CPU_STARTING phase no notifier shall return an error, because the
         * system is designed with the assumption that starting a CPU cannot
         * fail at this point. If an error happens here it will cause Xen to hit
         * the BUG_ON() in notify_cpu_starting(). In future, either this
         * notifier/enabling capabilities should be fixed to always return
         * success/void or notify_cpu_starting() and other common code should be
         * fixed to expect an error at CPU_STARTING phase.
         */
        ASSERT(system_state != SYS_STATE_boot);
        rc = enable_nonboot_cpu_caps(arm_errata);
        break;
    default:
        break;
    }

    return notifier_from_errno(rc);
}

static struct notifier_block cpu_errata_nfb = {
    .notifier_call = cpu_errata_callback,
};

static int __init cpu_errata_notifier_init(void)
{
    register_cpu_notifier(&cpu_errata_nfb);

    return 0;
}
/*
 * Initialization has to be done at init rather than presmp_init phase because
 * the callback should execute only after the secondary CPUs are initially
 * booted (in hotplug scenarios when the system state is not boot). On boot,
 * the enabling of errata workarounds will be triggered by the boot CPU from
 * start_xen().
 */
__initcall(cpu_errata_notifier_init);

/*
 * Local variables:
 * mode: C
 * c-file-style: "BSD"
 * c-basic-offset: 4
 * indent-tabs-mode: nil
 * End:
 */
