mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2026-04-03 12:05:13 +02:00
a84d30e8d2
[ Upstream commit75ed00055c] The Intel VT-d Scalable Mode PASID table entry consists of 512 bits (64 bytes). When tearing down an entry, the current implementation zeros the entire 64-byte structure immediately using multiple 64-bit writes. Since the IOMMU hardware may fetch these 64 bytes using multiple internal transactions (e.g., four 128-bit bursts), updating or zeroing the entire entry while it is active (P=1) risks a "torn" read. If a hardware fetch occurs simultaneously with the CPU zeroing the entry, the hardware could observe an inconsistent state, leading to unpredictable behavior or spurious faults. Follow the "Guidance to Software for Invalidations" in the VT-d spec (Section 6.5.3.3) by implementing the recommended ownership handshake: 1. Clear only the 'Present' (P) bit of the PASID entry. 2. Use a dma_wmb() to ensure the cleared bit is visible to hardware before proceeding. 3. Execute the required invalidation sequence (PASID cache, IOTLB, and Device-TLB flush) to ensure the hardware has released all cached references. 4. Only after the flushes are complete, zero out the remaining fields of the PASID entry. Also, add a dma_wmb() in pasid_set_present() to ensure that all other fields of the PASID entry are visible to the hardware before the Present bit is set. Fixes:0bbeb01a4f("iommu/vt-d: Manage scalalble mode PASID tables") Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Reviewed-by: Dmytro Maluka <dmaluka@chromium.org> Reviewed-by: Samiullah Khawaja <skhawaja@google.com> Reviewed-by: Kevin Tian <kevin.tian@intel.com> Link: https://lore.kernel.org/r/20260120061816.2132558-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel <joerg.roedel@amd.com> Signed-off-by: Sasha Levin <sashal@kernel.org>
328 lines
8.4 KiB
C
328 lines
8.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* pasid.h - PASID idr, table and entry header
|
|
*
|
|
* Copyright (C) 2018 Intel Corporation
|
|
*
|
|
* Author: Lu Baolu <baolu.lu@linux.intel.com>
|
|
*/
|
|
|
|
#ifndef __INTEL_PASID_H
|
|
#define __INTEL_PASID_H
|
|
|
|
#define PASID_MAX 0x100000
|
|
#define PASID_PTE_MASK 0x3F
|
|
#define PASID_PTE_PRESENT 1
|
|
#define PASID_PTE_FPD 2
|
|
#define PDE_PFN_MASK PAGE_MASK
|
|
#define PASID_PDE_SHIFT 6
|
|
#define MAX_NR_PASID_BITS 20
|
|
#define PASID_TBL_ENTRIES BIT(PASID_PDE_SHIFT)
|
|
|
|
#define is_pasid_enabled(entry) (((entry)->lo >> 3) & 0x1)
|
|
#define get_pasid_dir_size(entry) (1 << ((((entry)->lo >> 9) & 0x7) + 7))
|
|
|
|
/*
|
|
* Domain ID reserved for pasid entries programmed for first-level
|
|
* only and pass-through transfer modes.
|
|
*/
|
|
#define FLPT_DEFAULT_DID 1
|
|
#define NUM_RESERVED_DID 2
|
|
|
|
#define PASID_FLAG_NESTED BIT(1)
|
|
#define PASID_FLAG_PAGE_SNOOP BIT(2)
|
|
|
|
/*
|
|
* The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first-
|
|
* level translation, otherwise, 4-level paging will be used.
|
|
*/
|
|
#define PASID_FLAG_FL5LP BIT(1)
|
|
|
|
struct pasid_dir_entry {
|
|
u64 val;
|
|
};
|
|
|
|
struct pasid_entry {
|
|
u64 val[8];
|
|
};
|
|
|
|
#define PASID_ENTRY_PGTT_FL_ONLY (1)
|
|
#define PASID_ENTRY_PGTT_SL_ONLY (2)
|
|
#define PASID_ENTRY_PGTT_NESTED (3)
|
|
#define PASID_ENTRY_PGTT_PT (4)
|
|
|
|
/* The representative of a PASID table */
|
|
struct pasid_table {
|
|
void *table; /* pasid table pointer */
|
|
int order; /* page order of pasid table */
|
|
u32 max_pasid; /* max pasid */
|
|
};
|
|
|
|
/* Get PRESENT bit of a PASID directory entry. */
|
|
static inline bool pasid_pde_is_present(struct pasid_dir_entry *pde)
|
|
{
|
|
return READ_ONCE(pde->val) & PASID_PTE_PRESENT;
|
|
}
|
|
|
|
/* Get PASID table from a PASID directory entry. */
|
|
static inline struct pasid_entry *
|
|
get_pasid_table_from_pde(struct pasid_dir_entry *pde)
|
|
{
|
|
if (!pasid_pde_is_present(pde))
|
|
return NULL;
|
|
|
|
return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK);
|
|
}
|
|
|
|
/* Get PRESENT bit of a PASID table entry. */
|
|
static inline bool pasid_pte_is_present(struct pasid_entry *pte)
|
|
{
|
|
return READ_ONCE(pte->val[0]) & PASID_PTE_PRESENT;
|
|
}
|
|
|
|
/* Get PGTT field of a PASID table entry */
|
|
static inline u16 pasid_pte_get_pgtt(struct pasid_entry *pte)
|
|
{
|
|
return (u16)((READ_ONCE(pte->val[0]) >> 6) & 0x7);
|
|
}
|
|
|
|
static inline void pasid_clear_entry(struct pasid_entry *pe)
|
|
{
|
|
WRITE_ONCE(pe->val[0], 0);
|
|
WRITE_ONCE(pe->val[1], 0);
|
|
WRITE_ONCE(pe->val[2], 0);
|
|
WRITE_ONCE(pe->val[3], 0);
|
|
WRITE_ONCE(pe->val[4], 0);
|
|
WRITE_ONCE(pe->val[5], 0);
|
|
WRITE_ONCE(pe->val[6], 0);
|
|
WRITE_ONCE(pe->val[7], 0);
|
|
}
|
|
|
|
static inline void pasid_clear_entry_with_fpd(struct pasid_entry *pe)
|
|
{
|
|
WRITE_ONCE(pe->val[0], PASID_PTE_FPD);
|
|
WRITE_ONCE(pe->val[1], 0);
|
|
WRITE_ONCE(pe->val[2], 0);
|
|
WRITE_ONCE(pe->val[3], 0);
|
|
WRITE_ONCE(pe->val[4], 0);
|
|
WRITE_ONCE(pe->val[5], 0);
|
|
WRITE_ONCE(pe->val[6], 0);
|
|
WRITE_ONCE(pe->val[7], 0);
|
|
}
|
|
|
|
static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits)
|
|
{
|
|
u64 old;
|
|
|
|
old = READ_ONCE(*ptr);
|
|
WRITE_ONCE(*ptr, (old & ~mask) | bits);
|
|
}
|
|
|
|
static inline u64 pasid_get_bits(u64 *ptr)
|
|
{
|
|
return READ_ONCE(*ptr);
|
|
}
|
|
|
|
/*
|
|
* Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode
|
|
* PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_domain_id(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value);
|
|
}
|
|
|
|
/*
|
|
* Get domain ID value of a scalable mode PASID entry.
|
|
*/
|
|
static inline u16
|
|
pasid_get_domain_id(struct pasid_entry *pe)
|
|
{
|
|
return (u16)(READ_ONCE(pe->val[1]) & GENMASK_ULL(15, 0));
|
|
}
|
|
|
|
/*
|
|
* Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63)
|
|
* of a scalable mode PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_slptr(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value);
|
|
}
|
|
|
|
/*
|
|
* Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID
|
|
* entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_address_width(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2);
|
|
}
|
|
|
|
/*
|
|
* Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8)
|
|
* of a scalable mode PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_translation_type(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6);
|
|
}
|
|
|
|
/*
|
|
* Enable fault processing by clearing the FPD(Fault Processing
|
|
* Disable) field (Bit 1) of a scalable mode PASID entry.
|
|
*/
|
|
static inline void pasid_set_fault_enable(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[0], 1 << 1, 0);
|
|
}
|
|
|
|
/*
|
|
* Enable second level A/D bits by setting the SLADE (Second Level
|
|
* Access Dirty Enable) field (Bit 9) of a scalable mode PASID
|
|
* entry.
|
|
*/
|
|
static inline void pasid_set_ssade(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[0], 1 << 9, 1 << 9);
|
|
}
|
|
|
|
/*
|
|
* Disable second level A/D bits by clearing the SLADE (Second Level
|
|
* Access Dirty Enable) field (Bit 9) of a scalable mode PASID
|
|
* entry.
|
|
*/
|
|
static inline void pasid_clear_ssade(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[0], 1 << 9, 0);
|
|
}
|
|
|
|
/*
|
|
* Checks if second level A/D bits specifically the SLADE (Second Level
|
|
* Access Dirty Enable) field (Bit 9) of a scalable mode PASID
|
|
* entry is set.
|
|
*/
|
|
static inline bool pasid_get_ssade(struct pasid_entry *pe)
|
|
{
|
|
return pasid_get_bits(&pe->val[0]) & (1 << 9);
|
|
}
|
|
|
|
/*
|
|
* Setup the SRE(Supervisor Request Enable) field (Bit 128) of a
|
|
* scalable mode PASID entry.
|
|
*/
|
|
static inline void pasid_set_sre(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[2], 1 << 0, 1);
|
|
}
|
|
|
|
/*
|
|
* Setup the WPE(Write Protect Enable) field (Bit 132) of a
|
|
* scalable mode PASID entry.
|
|
*/
|
|
static inline void pasid_set_wpe(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[2], 1 << 4, 1 << 4);
|
|
}
|
|
|
|
/*
|
|
* Setup the P(Present) field (Bit 0) of a scalable mode PASID
|
|
* entry.
|
|
*/
|
|
static inline void pasid_set_present(struct pasid_entry *pe)
|
|
{
|
|
dma_wmb();
|
|
pasid_set_bits(&pe->val[0], 1 << 0, 1);
|
|
}
|
|
|
|
/*
|
|
* Clear the Present (P) bit (bit 0) of a scalable-mode PASID table entry.
|
|
* This initiates the transition of the entry's ownership from hardware
|
|
* to software. The caller is responsible for fulfilling the invalidation
|
|
* handshake recommended by the VT-d spec, Section 6.5.3.3 (Guidance to
|
|
* Software for Invalidations).
|
|
*/
|
|
static inline void pasid_clear_present(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[0], 1 << 0, 0);
|
|
dma_wmb();
|
|
}
|
|
|
|
/*
|
|
* Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID
|
|
* entry.
|
|
*/
|
|
static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value)
|
|
{
|
|
pasid_set_bits(&pe->val[1], 1 << 23, value << 23);
|
|
}
|
|
|
|
/*
|
|
* Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode
|
|
* PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_pgsnp(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24);
|
|
}
|
|
|
|
/*
|
|
* Setup the First Level Page table Pointer field (Bit 140~191)
|
|
* of a scalable mode PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_flptr(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[2], VTD_PAGE_MASK, value);
|
|
}
|
|
|
|
/*
|
|
* Setup the First Level Paging Mode field (Bit 130~131) of a
|
|
* scalable mode PASID entry.
|
|
*/
|
|
static inline void
|
|
pasid_set_flpm(struct pasid_entry *pe, u64 value)
|
|
{
|
|
pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2);
|
|
}
|
|
|
|
/*
|
|
* Setup the Extended Access Flag Enable (EAFE) field (Bit 135)
|
|
* of a scalable mode PASID entry.
|
|
*/
|
|
static inline void pasid_set_eafe(struct pasid_entry *pe)
|
|
{
|
|
pasid_set_bits(&pe->val[2], 1 << 7, 1 << 7);
|
|
}
|
|
|
|
extern unsigned int intel_pasid_max_id;
|
|
int intel_pasid_alloc_table(struct device *dev);
|
|
void intel_pasid_free_table(struct device *dev);
|
|
struct pasid_table *intel_pasid_get_table(struct device *dev);
|
|
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
|
|
struct device *dev, pgd_t *pgd,
|
|
u32 pasid, u16 did, int flags);
|
|
int intel_pasid_setup_second_level(struct intel_iommu *iommu,
|
|
struct dmar_domain *domain,
|
|
struct device *dev, u32 pasid);
|
|
int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,
|
|
struct device *dev, u32 pasid,
|
|
bool enabled);
|
|
int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
|
|
struct device *dev, u32 pasid);
|
|
int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
|
|
u32 pasid, struct dmar_domain *domain);
|
|
void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
|
|
struct device *dev, u32 pasid,
|
|
bool fault_ignore);
|
|
void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,
|
|
struct device *dev, u32 pasid);
|
|
int intel_pasid_setup_sm_context(struct device *dev);
|
|
void intel_pasid_teardown_sm_context(struct device *dev);
|
|
#endif /* __INTEL_PASID_H */
|