RISC-V Patches for the 6.6 Merge Window, Part 2 (try 2)
* The kernel now dynamically probes for misaligned access speed, as opposed to relying on a table of known implementations. * Support for non-coherent devices on systems using the Andes AX45MP core, including the RZ/Five SoCs. * Support for the V extension in ptrace(), again. * Support for KASLR. * Support for the BPF prog pack allocator in RISC-V. * A handful of bug fixes and cleanups. -----BEGIN PGP SIGNATURE----- iQJHBAABCAAxFiEEKzw3R0RoQ7JKlDp6LhMZ81+7GIkFAmT8eV0THHBhbG1lckBk YWJiZWx0LmNvbQAKCRAuExnzX7sYiQYTD/9V6asKMDdWUV+gti/gRvJsiYUjIrrK h4MB8hL3fHfCLBpTD4rU6K1Gx6hzPjGsxIuQyAq/hf752KB/9XUiIVziRBv2ZEBb GuTFCXfg0QXBUlxBZzFw5SKUuKXgRaMAQ14qjy3tfLk31YMQmBtAlEPdDM8mZOCQ zNI3bbdn6zASeaSMh7hwBoOJWP2ACoOEW7RcD44EDT8jb3YW5rEF86x0XtYLgJb6 xhaR4ieIdaOLxz2RbjXj0GcPIBfhTxZbwN3fLlD8PxuGqCKn5kN03bPPwP9tMTAc z02EgVcSDvJWpYikuuTkPMxpSi18OZPJ6eriwOv5ccP5NXQScO09iGo7IZEM7OzO j1IrIXyncU4BhxlpWombU454Va+ezUlfh9uh+MrJ+Bnve3T3S9ax7AV4S8vkJZlT bnmJVS/g7L/7nxTQdJ3zoAo2WzFQXL0C8SR5tGo/3aRk0uYoliHy/W419f55F9GZ rFcc+LMqai8N4bLN3whaK0NnuodNWHoNlpcd/5ncJwecswuDkah3LWcd4rwBrWhu 8RIkIfpdr/vTQjUVXVLeMHdKB+lST3iF1feMqJj0PfTyvTZi5yfSppjAfkAdVq+9 lHqAjsaGdiCrOtLxb0oBR2PTDQPAm2gN2meuSMommDQR6Vul8K5WcQml9Zx9QEWA eDXWYDZISKYJbA== =s89m -----END PGP SIGNATURE----- Merge tag 'riscv-for-linus-6.6-mw2-2' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux Pull more RISC-V updates from Palmer Dabbelt: - The kernel now dynamically probes for misaligned access speed, as opposed to relying on a table of known implementations. - Support for non-coherent devices on systems using the Andes AX45MP core, including the RZ/Five SoCs. - Support for the V extension in ptrace(), again. - Support for KASLR. - Support for the BPF prog pack allocator in RISC-V. - A handful of bug fixes and cleanups. * tag 'riscv-for-linus-6.6-mw2-2' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (25 commits) soc: renesas: Kconfig: For ARCH_R9A07G043 select the required configs if dependencies are met riscv: Kconfig.errata: Add dependency for RISCV_SBI in ERRATA_ANDES config riscv: Kconfig.errata: Drop dependency for MMU in ERRATA_ANDES_CMO config riscv: Kconfig: Select DMA_DIRECT_REMAP only if MMU is enabled bpf, riscv: use prog pack allocator in the BPF JIT riscv: implement a memset like function for text riscv: extend patch_text_nosync() for multiple pages bpf: make bpf_prog_pack allocator portable riscv: libstub: Implement KASLR by using generic functions libstub: Fix compilation warning for rv32 arm64: libstub: Move KASLR handling functions to kaslr.c riscv: Dump out kernel offset information on panic riscv: Introduce virtual kernel mapping KASLR RISC-V: Add ptrace support for vectors soc: renesas: Kconfig: Select the required configs for RZ/Five SoC cache: Add L2 cache management for Andes AX45MP RISC-V core dt-bindings: cache: andestech,ax45mp-cache: Add DT binding documentation for L2 cache controller riscv: mm: dma-noncoherent: nonstandard cache operations support riscv: errata: Add Andes alternative ports riscv: asm: vendorid_list: Add Andes Technology to the vendors list ...
This commit is contained in:
Linus Torvalds
commit
1b37a0a2d4
|
|
@ -0,0 +1,81 @@
|
|||
# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
|
||||
# Copyright (C) 2023 Renesas Electronics Corp.
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/cache/andestech,ax45mp-cache.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: Andestech AX45MP L2 Cache Controller
|
||||
|
||||
maintainers:
|
||||
- Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
|
||||
|
||||
description:
|
||||
A level-2 cache (L2C) is used to improve the system performance by providing
|
||||
a large amount of cache line entries and reasonable access delays. The L2C
|
||||
is shared between cores, and a non-inclusive non-exclusive policy is used.
|
||||
|
||||
select:
|
||||
properties:
|
||||
compatible:
|
||||
contains:
|
||||
enum:
|
||||
- andestech,ax45mp-cache
|
||||
|
||||
required:
|
||||
- compatible
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
items:
|
||||
- const: andestech,ax45mp-cache
|
||||
- const: cache
|
||||
|
||||
reg:
|
||||
maxItems: 1
|
||||
|
||||
interrupts:
|
||||
maxItems: 1
|
||||
|
||||
cache-line-size:
|
||||
const: 64
|
||||
|
||||
cache-level:
|
||||
const: 2
|
||||
|
||||
cache-sets:
|
||||
const: 1024
|
||||
|
||||
cache-size:
|
||||
enum: [131072, 262144, 524288, 1048576, 2097152]
|
||||
|
||||
cache-unified: true
|
||||
|
||||
next-level-cache: true
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- interrupts
|
||||
- cache-line-size
|
||||
- cache-level
|
||||
- cache-sets
|
||||
- cache-size
|
||||
- cache-unified
|
||||
|
||||
examples:
|
||||
- |
|
||||
#include <dt-bindings/interrupt-controller/irq.h>
|
||||
|
||||
cache-controller@2010000 {
|
||||
compatible = "andestech,ax45mp-cache", "cache";
|
||||
reg = <0x13400000 0x100000>;
|
||||
interrupts = <508 IRQ_TYPE_LEVEL_HIGH>;
|
||||
cache-line-size = <64>;
|
||||
cache-level = <2>;
|
||||
cache-sets = <1024>;
|
||||
cache-size = <262144>;
|
||||
cache-unified;
|
||||
};
|
||||
|
|
@ -87,13 +87,12 @@ The following keys are defined:
|
|||
emulated via software, either in or below the kernel. These accesses are
|
||||
always extremely slow.
|
||||
|
||||
* :c:macro:`RISCV_HWPROBE_MISALIGNED_SLOW`: Misaligned accesses are supported
|
||||
in hardware, but are slower than the corresponding aligned accesses
|
||||
sequences.
|
||||
* :c:macro:`RISCV_HWPROBE_MISALIGNED_SLOW`: Misaligned accesses are slower
|
||||
than equivalent byte accesses. Misaligned accesses may be supported
|
||||
directly in hardware, or trapped and emulated by software.
|
||||
|
||||
* :c:macro:`RISCV_HWPROBE_MISALIGNED_FAST`: Misaligned accesses are supported
|
||||
in hardware and are faster than the corresponding aligned accesses
|
||||
sequences.
|
||||
* :c:macro:`RISCV_HWPROBE_MISALIGNED_FAST`: Misaligned accesses are faster
|
||||
than equivalent byte accesses.
|
||||
|
||||
* :c:macro:`RISCV_HWPROBE_MISALIGNED_UNSUPPORTED`: Misaligned accesses are
|
||||
not supported at all and will generate a misaligned address fault.
|
||||
|
|
|
|||
|
|
@ -20406,6 +20406,13 @@ S: Supported
|
|||
T: git git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git
|
||||
F: drivers/staging/
|
||||
|
||||
STANDALONE CACHE CONTROLLER DRIVERS
|
||||
M: Conor Dooley <conor@kernel.org>
|
||||
L: linux-riscv@lists.infradead.org
|
||||
S: Maintained
|
||||
T: git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/
|
||||
F: drivers/cache
|
||||
|
||||
STARFIRE/DURALAN NETWORK DRIVER
|
||||
M: Ion Badulescu <ionut@badula.org>
|
||||
S: Odd Fixes
|
||||
|
|
|
|||
|
|
@ -156,4 +156,6 @@ static inline void efi_capsule_flush_cache_range(void *addr, int size)
|
|||
|
||||
efi_status_t efi_handle_corrupted_x18(efi_status_t s, const char *f);
|
||||
|
||||
void efi_icache_sync(unsigned long start, unsigned long end);
|
||||
|
||||
#endif /* _ASM_EFI_H */
|
||||
|
|
|
|||
|
|
@ -273,7 +273,14 @@ config RISCV_DMA_NONCOHERENT
|
|||
select ARCH_HAS_SYNC_DMA_FOR_CPU
|
||||
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
|
||||
select DMA_BOUNCE_UNALIGNED_KMALLOC if SWIOTLB
|
||||
select DMA_DIRECT_REMAP
|
||||
select DMA_DIRECT_REMAP if MMU
|
||||
|
||||
config RISCV_NONSTANDARD_CACHE_OPS
|
||||
bool
|
||||
depends on RISCV_DMA_NONCOHERENT
|
||||
help
|
||||
This enables function pointer support for non-standard noncoherent
|
||||
systems to handle cache management.
|
||||
|
||||
config AS_HAS_INSN
|
||||
def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero)
|
||||
|
|
@ -713,6 +720,25 @@ config RELOCATABLE
|
|||
|
||||
If unsure, say N.
|
||||
|
||||
config RANDOMIZE_BASE
|
||||
bool "Randomize the address of the kernel image"
|
||||
select RELOCATABLE
|
||||
depends on MMU && 64BIT && !XIP_KERNEL
|
||||
help
|
||||
Randomizes the virtual address at which the kernel image is
|
||||
loaded, as a security feature that deters exploit attempts
|
||||
relying on knowledge of the location of kernel internals.
|
||||
|
||||
It is the bootloader's job to provide entropy, by passing a
|
||||
random u64 value in /chosen/kaslr-seed at kernel entry.
|
||||
|
||||
When booting via the UEFI stub, it will invoke the firmware's
|
||||
EFI_RNG_PROTOCOL implementation (if available) to supply entropy
|
||||
to the kernel proper. In addition, it will randomise the physical
|
||||
location of the kernel Image as well.
|
||||
|
||||
If unsure, say N.
|
||||
|
||||
endmenu # "Kernel features"
|
||||
|
||||
menu "Boot options"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,26 @@
|
|||
menu "CPU errata selection"
|
||||
|
||||
config ERRATA_ANDES
|
||||
bool "Andes AX45MP errata"
|
||||
depends on RISCV_ALTERNATIVE && RISCV_SBI
|
||||
help
|
||||
All Andes errata Kconfig depend on this Kconfig. Disabling
|
||||
this Kconfig will disable all Andes errata. Please say "Y"
|
||||
here if your platform uses Andes CPU cores.
|
||||
|
||||
Otherwise, please say "N" here to avoid unnecessary overhead.
|
||||
|
||||
config ERRATA_ANDES_CMO
|
||||
bool "Apply Andes cache management errata"
|
||||
depends on ERRATA_ANDES && ARCH_R9A07G043
|
||||
select RISCV_DMA_NONCOHERENT
|
||||
default y
|
||||
help
|
||||
This will apply the cache management errata to handle the
|
||||
non-standard handling on non-coherent operations on Andes cores.
|
||||
|
||||
If you don't know what to do here, say "Y".
|
||||
|
||||
config ERRATA_SIFIVE
|
||||
bool "SiFive errata"
|
||||
depends on RISCV_ALTERNATIVE
|
||||
|
|
|
|||
|
|
@ -2,5 +2,6 @@ ifdef CONFIG_RELOCATABLE
|
|||
KBUILD_CFLAGS += -fno-pie
|
||||
endif
|
||||
|
||||
obj-$(CONFIG_ERRATA_ANDES) += andes/
|
||||
obj-$(CONFIG_ERRATA_SIFIVE) += sifive/
|
||||
obj-$(CONFIG_ERRATA_THEAD) += thead/
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
obj-y += errata.o
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Erratas to be applied for Andes CPU cores
|
||||
*
|
||||
* Copyright (C) 2023 Renesas Electronics Corporation.
|
||||
*
|
||||
* Author: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
|
||||
*/
|
||||
|
||||
#include <linux/memory.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
#include <asm/alternative.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/errata_list.h>
|
||||
#include <asm/patch.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/sbi.h>
|
||||
#include <asm/vendorid_list.h>
|
||||
|
||||
#define ANDESTECH_AX45MP_MARCHID 0x8000000000008a45UL
|
||||
#define ANDESTECH_AX45MP_MIMPID 0x500UL
|
||||
#define ANDESTECH_SBI_EXT_ANDES 0x0900031E
|
||||
|
||||
#define ANDES_SBI_EXT_IOCP_SW_WORKAROUND 1
|
||||
|
||||
static long ax45mp_iocp_sw_workaround(void)
|
||||
{
|
||||
struct sbiret ret;
|
||||
|
||||
/*
|
||||
* ANDES_SBI_EXT_IOCP_SW_WORKAROUND SBI EXT checks if the IOCP is missing and
|
||||
* cache is controllable only then CMO will be applied to the platform.
|
||||
*/
|
||||
ret = sbi_ecall(ANDESTECH_SBI_EXT_ANDES, ANDES_SBI_EXT_IOCP_SW_WORKAROUND,
|
||||
0, 0, 0, 0, 0, 0);
|
||||
|
||||
return ret.error ? 0 : ret.value;
|
||||
}
|
||||
|
||||
static bool errata_probe_iocp(unsigned int stage, unsigned long arch_id, unsigned long impid)
|
||||
{
|
||||
if (!IS_ENABLED(CONFIG_ERRATA_ANDES_CMO))
|
||||
return false;
|
||||
|
||||
if (arch_id != ANDESTECH_AX45MP_MARCHID || impid != ANDESTECH_AX45MP_MIMPID)
|
||||
return false;
|
||||
|
||||
if (!ax45mp_iocp_sw_workaround())
|
||||
return false;
|
||||
|
||||
/* Set this just to make core cbo code happy */
|
||||
riscv_cbom_block_size = 1;
|
||||
riscv_noncoherent_supported();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void __init_or_module andes_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
||||
unsigned long archid, unsigned long impid,
|
||||
unsigned int stage)
|
||||
{
|
||||
errata_probe_iocp(stage, archid, impid);
|
||||
|
||||
/* we have nothing to patch here ATM so just return back */
|
||||
}
|
||||
|
|
@ -120,11 +120,3 @@ void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
|||
if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
|
||||
local_flush_icache_all();
|
||||
}
|
||||
|
||||
void thead_feature_probe_func(unsigned int cpu,
|
||||
unsigned long archid,
|
||||
unsigned long impid)
|
||||
{
|
||||
if ((archid == 0) && (impid == 0))
|
||||
per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_FAST;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@
|
|||
#define ALT_OLD_PTR(a) __ALT_PTR(a, old_offset)
|
||||
#define ALT_ALT_PTR(a) __ALT_PTR(a, alt_offset)
|
||||
|
||||
void probe_vendor_features(unsigned int cpu);
|
||||
void __init apply_boot_alternatives(void);
|
||||
void __init apply_early_boot_alternatives(void);
|
||||
void apply_module_alternatives(void *start, size_t length);
|
||||
|
|
@ -46,6 +45,9 @@ struct alt_entry {
|
|||
u32 patch_id; /* The patch ID (erratum ID or cpufeature ID) */
|
||||
};
|
||||
|
||||
void andes_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
||||
unsigned long archid, unsigned long impid,
|
||||
unsigned int stage);
|
||||
void sifive_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
||||
unsigned long archid, unsigned long impid,
|
||||
unsigned int stage);
|
||||
|
|
@ -53,15 +55,11 @@ void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
|||
unsigned long archid, unsigned long impid,
|
||||
unsigned int stage);
|
||||
|
||||
void thead_feature_probe_func(unsigned int cpu, unsigned long archid,
|
||||
unsigned long impid);
|
||||
|
||||
void riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end,
|
||||
unsigned int stage);
|
||||
|
||||
#else /* CONFIG_RISCV_ALTERNATIVE */
|
||||
|
||||
static inline void probe_vendor_features(unsigned int cpu) { }
|
||||
static inline void apply_boot_alternatives(void) { }
|
||||
static inline void apply_early_boot_alternatives(void) { }
|
||||
static inline void apply_module_alternatives(void *start, size_t length) { }
|
||||
|
|
|
|||
|
|
@ -30,4 +30,6 @@ DECLARE_PER_CPU(long, misaligned_access_speed);
|
|||
/* Per-cpu ISA extensions. */
|
||||
extern struct riscv_isainfo hart_isa[NR_CPUS];
|
||||
|
||||
void check_unaligned_access(int cpu);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2023 Renesas Electronics Corp.
|
||||
*/
|
||||
|
||||
#ifndef __ASM_DMA_NONCOHERENT_H
|
||||
#define __ASM_DMA_NONCOHERENT_H
|
||||
|
||||
#include <linux/dma-direct.h>
|
||||
|
||||
/*
|
||||
* struct riscv_nonstd_cache_ops - Structure for non-standard CMO function pointers
|
||||
*
|
||||
* @wback: Function pointer for cache writeback
|
||||
* @inv: Function pointer for invalidating cache
|
||||
* @wback_inv: Function pointer for flushing the cache (writeback + invalidating)
|
||||
*/
|
||||
struct riscv_nonstd_cache_ops {
|
||||
void (*wback)(phys_addr_t paddr, size_t size);
|
||||
void (*inv)(phys_addr_t paddr, size_t size);
|
||||
void (*wback_inv)(phys_addr_t paddr, size_t size);
|
||||
};
|
||||
|
||||
extern struct riscv_nonstd_cache_ops noncoherent_cache_ops;
|
||||
|
||||
void riscv_noncoherent_register_cache_ops(const struct riscv_nonstd_cache_ops *ops);
|
||||
|
||||
#endif /* __ASM_DMA_NONCOHERENT_H */
|
||||
|
|
@ -45,4 +45,6 @@ void arch_efi_call_virt_teardown(void);
|
|||
|
||||
unsigned long stext_offset(void);
|
||||
|
||||
void efi_icache_sync(unsigned long start, unsigned long end);
|
||||
|
||||
#endif /* _ASM_EFI_H */
|
||||
|
|
|
|||
|
|
@ -11,6 +11,11 @@
|
|||
#include <asm/hwcap.h>
|
||||
#include <asm/vendorid_list.h>
|
||||
|
||||
#ifdef CONFIG_ERRATA_ANDES
|
||||
#define ERRATA_ANDESTECH_NO_IOCP 0
|
||||
#define ERRATA_ANDESTECH_NUMBER 1
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_ERRATA_SIFIVE
|
||||
#define ERRATA_SIFIVE_CIP_453 0
|
||||
#define ERRATA_SIFIVE_CIP_1200 1
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ typedef struct page *pgtable_t;
|
|||
struct kernel_mapping {
|
||||
unsigned long page_offset;
|
||||
unsigned long virt_addr;
|
||||
unsigned long virt_offset;
|
||||
uintptr_t phys_addr;
|
||||
uintptr_t size;
|
||||
/* Offset between linear mapping virtual address and kernel load address */
|
||||
|
|
@ -185,6 +186,8 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x);
|
|||
|
||||
#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x))
|
||||
|
||||
unsigned long kaslr_offset(void);
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#define virt_addr_valid(vaddr) ({ \
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#define _ASM_RISCV_PATCH_H
|
||||
|
||||
int patch_text_nosync(void *addr, const void *insns, size_t len);
|
||||
int patch_text_set_nosync(void *addr, u8 c, size_t len);
|
||||
int patch_text(void *addr, u32 *insns, int ninsns);
|
||||
|
||||
extern int riscv_patch_in_stop_machine;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#ifndef ASM_VENDOR_LIST_H
|
||||
#define ASM_VENDOR_LIST_H
|
||||
|
||||
#define ANDESTECH_VENDOR_ID 0x31e
|
||||
#define SIFIVE_VENDOR_ID 0x489
|
||||
#define THEAD_VENDOR_ID 0x5b7
|
||||
|
||||
|
|
|
|||
|
|
@ -108,13 +108,18 @@ struct __riscv_v_ext_state {
|
|||
* In signal handler, datap will be set a correct user stack offset
|
||||
* and vector registers will be copied to the address of datap
|
||||
* pointer.
|
||||
*
|
||||
* In ptrace syscall, datap will be set to zero and the vector
|
||||
* registers will be copied to the address right after this
|
||||
* structure.
|
||||
*/
|
||||
};
|
||||
|
||||
struct __riscv_v_regset_state {
|
||||
unsigned long vstart;
|
||||
unsigned long vl;
|
||||
unsigned long vtype;
|
||||
unsigned long vcsr;
|
||||
unsigned long vlenb;
|
||||
char vreg[];
|
||||
};
|
||||
|
||||
/*
|
||||
* According to spec: The number of bits in a single vector register,
|
||||
* VLEN >= ELEN, which must be a power of 2, and must be no greater than
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ extra-y += vmlinux.lds
|
|||
obj-y += head.o
|
||||
obj-y += soc.o
|
||||
obj-$(CONFIG_RISCV_ALTERNATIVE) += alternative.o
|
||||
obj-y += copy-unaligned.o
|
||||
obj-y += cpu.o
|
||||
obj-y += cpufeature.o
|
||||
obj-y += entry.o
|
||||
|
|
|
|||
|
|
@ -27,8 +27,6 @@ struct cpu_manufacturer_info_t {
|
|||
void (*patch_func)(struct alt_entry *begin, struct alt_entry *end,
|
||||
unsigned long archid, unsigned long impid,
|
||||
unsigned int stage);
|
||||
void (*feature_probe_func)(unsigned int cpu, unsigned long archid,
|
||||
unsigned long impid);
|
||||
};
|
||||
|
||||
static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info)
|
||||
|
|
@ -43,8 +41,12 @@ static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info
|
|||
cpu_mfr_info->imp_id = sbi_get_mimpid();
|
||||
#endif
|
||||
|
||||
cpu_mfr_info->feature_probe_func = NULL;
|
||||
switch (cpu_mfr_info->vendor_id) {
|
||||
#ifdef CONFIG_ERRATA_ANDES
|
||||
case ANDESTECH_VENDOR_ID:
|
||||
cpu_mfr_info->patch_func = andes_errata_patch_func;
|
||||
break;
|
||||
#endif
|
||||
#ifdef CONFIG_ERRATA_SIFIVE
|
||||
case SIFIVE_VENDOR_ID:
|
||||
cpu_mfr_info->patch_func = sifive_errata_patch_func;
|
||||
|
|
@ -53,7 +55,6 @@ static void riscv_fill_cpu_mfr_info(struct cpu_manufacturer_info_t *cpu_mfr_info
|
|||
#ifdef CONFIG_ERRATA_THEAD
|
||||
case THEAD_VENDOR_ID:
|
||||
cpu_mfr_info->patch_func = thead_errata_patch_func;
|
||||
cpu_mfr_info->feature_probe_func = thead_feature_probe_func;
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
|
|
@ -143,20 +144,6 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len,
|
|||
}
|
||||
}
|
||||
|
||||
/* Called on each CPU as it starts */
|
||||
void probe_vendor_features(unsigned int cpu)
|
||||
{
|
||||
struct cpu_manufacturer_info_t cpu_mfr_info;
|
||||
|
||||
riscv_fill_cpu_mfr_info(&cpu_mfr_info);
|
||||
if (!cpu_mfr_info.feature_probe_func)
|
||||
return;
|
||||
|
||||
cpu_mfr_info.feature_probe_func(cpu,
|
||||
cpu_mfr_info.arch_id,
|
||||
cpu_mfr_info.imp_id);
|
||||
}
|
||||
|
||||
/*
|
||||
* This is called very early in the boot process (directly after we run
|
||||
* a feature detect on the boot CPU). No need to worry about other CPUs
|
||||
|
|
@ -211,7 +198,6 @@ void __init apply_boot_alternatives(void)
|
|||
/* If called on non-boot cpu things could go wrong */
|
||||
WARN_ON(smp_processor_id() != 0);
|
||||
|
||||
probe_vendor_features(0);
|
||||
_apply_alternatives((struct alt_entry *)__alt_start,
|
||||
(struct alt_entry *)__alt_end,
|
||||
RISCV_ALTERNATIVES_BOOT);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (C) 2023 Rivos Inc. */
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/asm.h>
|
||||
|
||||
.text
|
||||
|
||||
/* void __riscv_copy_words_unaligned(void *, const void *, size_t) */
|
||||
/* Performs a memcpy without aligning buffers, using word loads and stores. */
|
||||
/* Note: The size is truncated to a multiple of 8 * SZREG */
|
||||
ENTRY(__riscv_copy_words_unaligned)
|
||||
andi a4, a2, ~((8*SZREG)-1)
|
||||
beqz a4, 2f
|
||||
add a3, a1, a4
|
||||
1:
|
||||
REG_L a4, 0(a1)
|
||||
REG_L a5, SZREG(a1)
|
||||
REG_L a6, 2*SZREG(a1)
|
||||
REG_L a7, 3*SZREG(a1)
|
||||
REG_L t0, 4*SZREG(a1)
|
||||
REG_L t1, 5*SZREG(a1)
|
||||
REG_L t2, 6*SZREG(a1)
|
||||
REG_L t3, 7*SZREG(a1)
|
||||
REG_S a4, 0(a0)
|
||||
REG_S a5, SZREG(a0)
|
||||
REG_S a6, 2*SZREG(a0)
|
||||
REG_S a7, 3*SZREG(a0)
|
||||
REG_S t0, 4*SZREG(a0)
|
||||
REG_S t1, 5*SZREG(a0)
|
||||
REG_S t2, 6*SZREG(a0)
|
||||
REG_S t3, 7*SZREG(a0)
|
||||
addi a0, a0, 8*SZREG
|
||||
addi a1, a1, 8*SZREG
|
||||
bltu a1, a3, 1b
|
||||
|
||||
2:
|
||||
ret
|
||||
END(__riscv_copy_words_unaligned)
|
||||
|
||||
/* void __riscv_copy_bytes_unaligned(void *, const void *, size_t) */
|
||||
/* Performs a memcpy without aligning buffers, using only byte accesses. */
|
||||
/* Note: The size is truncated to a multiple of 8 */
|
||||
ENTRY(__riscv_copy_bytes_unaligned)
|
||||
andi a4, a2, ~(8-1)
|
||||
beqz a4, 2f
|
||||
add a3, a1, a4
|
||||
1:
|
||||
lb a4, 0(a1)
|
||||
lb a5, 1(a1)
|
||||
lb a6, 2(a1)
|
||||
lb a7, 3(a1)
|
||||
lb t0, 4(a1)
|
||||
lb t1, 5(a1)
|
||||
lb t2, 6(a1)
|
||||
lb t3, 7(a1)
|
||||
sb a4, 0(a0)
|
||||
sb a5, 1(a0)
|
||||
sb a6, 2(a0)
|
||||
sb a7, 3(a0)
|
||||
sb t0, 4(a0)
|
||||
sb t1, 5(a0)
|
||||
sb t2, 6(a0)
|
||||
sb t3, 7(a0)
|
||||
addi a0, a0, 8
|
||||
addi a1, a1, 8
|
||||
bltu a1, a3, 1b
|
||||
|
||||
2:
|
||||
ret
|
||||
END(__riscv_copy_bytes_unaligned)
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Copyright (C) 2023 Rivos, Inc.
|
||||
*/
|
||||
#ifndef __RISCV_KERNEL_COPY_UNALIGNED_H
|
||||
#define __RISCV_KERNEL_COPY_UNALIGNED_H
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
void __riscv_copy_words_unaligned(void *dst, const void *src, size_t size);
|
||||
void __riscv_copy_bytes_unaligned(void *dst, const void *src, size_t size);
|
||||
|
||||
#endif /* __RISCV_KERNEL_COPY_UNALIGNED_H */
|
||||
|
|
@ -18,12 +18,19 @@
|
|||
#include <asm/cacheflush.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/hwcap.h>
|
||||
#include <asm/hwprobe.h>
|
||||
#include <asm/patch.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/vector.h>
|
||||
|
||||
#include "copy-unaligned.h"
|
||||
|
||||
#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
|
||||
|
||||
#define MISALIGNED_ACCESS_JIFFIES_LG2 1
|
||||
#define MISALIGNED_BUFFER_SIZE 0x4000
|
||||
#define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
|
||||
|
||||
unsigned long elf_hwcap __read_mostly;
|
||||
|
||||
/* Host ISA bitmap */
|
||||
|
|
@ -549,6 +556,103 @@ unsigned long riscv_get_elf_hwcap(void)
|
|||
return hwcap;
|
||||
}
|
||||
|
||||
void check_unaligned_access(int cpu)
|
||||
{
|
||||
u64 start_cycles, end_cycles;
|
||||
u64 word_cycles;
|
||||
u64 byte_cycles;
|
||||
int ratio;
|
||||
unsigned long start_jiffies, now;
|
||||
struct page *page;
|
||||
void *dst;
|
||||
void *src;
|
||||
long speed = RISCV_HWPROBE_MISALIGNED_SLOW;
|
||||
|
||||
page = alloc_pages(GFP_NOWAIT, get_order(MISALIGNED_BUFFER_SIZE));
|
||||
if (!page) {
|
||||
pr_warn("Can't alloc pages to measure memcpy performance");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Make an unaligned destination buffer. */
|
||||
dst = (void *)((unsigned long)page_address(page) | 0x1);
|
||||
/* Unalign src as well, but differently (off by 1 + 2 = 3). */
|
||||
src = dst + (MISALIGNED_BUFFER_SIZE / 2);
|
||||
src += 2;
|
||||
word_cycles = -1ULL;
|
||||
/* Do a warmup. */
|
||||
__riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
|
||||
preempt_disable();
|
||||
start_jiffies = jiffies;
|
||||
while ((now = jiffies) == start_jiffies)
|
||||
cpu_relax();
|
||||
|
||||
/*
|
||||
* For a fixed amount of time, repeatedly try the function, and take
|
||||
* the best time in cycles as the measurement.
|
||||
*/
|
||||
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
|
||||
start_cycles = get_cycles64();
|
||||
/* Ensure the CSR read can't reorder WRT to the copy. */
|
||||
mb();
|
||||
__riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
|
||||
/* Ensure the copy ends before the end time is snapped. */
|
||||
mb();
|
||||
end_cycles = get_cycles64();
|
||||
if ((end_cycles - start_cycles) < word_cycles)
|
||||
word_cycles = end_cycles - start_cycles;
|
||||
}
|
||||
|
||||
byte_cycles = -1ULL;
|
||||
__riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
|
||||
start_jiffies = jiffies;
|
||||
while ((now = jiffies) == start_jiffies)
|
||||
cpu_relax();
|
||||
|
||||
while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
|
||||
start_cycles = get_cycles64();
|
||||
mb();
|
||||
__riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
|
||||
mb();
|
||||
end_cycles = get_cycles64();
|
||||
if ((end_cycles - start_cycles) < byte_cycles)
|
||||
byte_cycles = end_cycles - start_cycles;
|
||||
}
|
||||
|
||||
preempt_enable();
|
||||
|
||||
/* Don't divide by zero. */
|
||||
if (!word_cycles || !byte_cycles) {
|
||||
pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n",
|
||||
cpu);
|
||||
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (word_cycles < byte_cycles)
|
||||
speed = RISCV_HWPROBE_MISALIGNED_FAST;
|
||||
|
||||
ratio = div_u64((byte_cycles * 100), word_cycles);
|
||||
pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n",
|
||||
cpu,
|
||||
ratio / 100,
|
||||
ratio % 100,
|
||||
(speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow");
|
||||
|
||||
per_cpu(misaligned_access_speed, cpu) = speed;
|
||||
|
||||
out:
|
||||
__free_pages(page, get_order(MISALIGNED_BUFFER_SIZE));
|
||||
}
|
||||
|
||||
static int check_unaligned_access_boot_cpu(void)
|
||||
{
|
||||
check_unaligned_access(0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
arch_initcall(check_unaligned_access_boot_cpu);
|
||||
|
||||
#ifdef CONFIG_RISCV_ALTERNATIVE
|
||||
/*
|
||||
* Alternative patch sites consider 48 bits when determining when to patch
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ __efistub__start = _start;
|
|||
__efistub__start_kernel = _start_kernel;
|
||||
__efistub__end = _end;
|
||||
__efistub__edata = _edata;
|
||||
__efistub___init_text_end = __init_text_end;
|
||||
__efistub_screen_info = screen_info;
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include <linux/spinlock.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/memory.h>
|
||||
#include <linux/string.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <asm/kprobes.h>
|
||||
|
|
@ -53,12 +54,51 @@ static void patch_unmap(int fixmap)
|
|||
}
|
||||
NOKPROBE_SYMBOL(patch_unmap);
|
||||
|
||||
static int patch_insn_write(void *addr, const void *insn, size_t len)
|
||||
static int __patch_insn_set(void *addr, u8 c, size_t len)
|
||||
{
|
||||
void *waddr = addr;
|
||||
bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE;
|
||||
|
||||
/*
|
||||
* Only two pages can be mapped at a time for writing.
|
||||
*/
|
||||
if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
|
||||
return -EINVAL;
|
||||
/*
|
||||
* Before reaching here, it was expected to lock the text_mutex
|
||||
* already, so we don't need to give another lock here and could
|
||||
* ensure that it was safe between each cores.
|
||||
*/
|
||||
lockdep_assert_held(&text_mutex);
|
||||
|
||||
if (across_pages)
|
||||
patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
|
||||
|
||||
waddr = patch_map(addr, FIX_TEXT_POKE0);
|
||||
|
||||
memset(waddr, c, len);
|
||||
|
||||
patch_unmap(FIX_TEXT_POKE0);
|
||||
|
||||
if (across_pages)
|
||||
patch_unmap(FIX_TEXT_POKE1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
NOKPROBE_SYMBOL(__patch_insn_set);
|
||||
|
||||
static int __patch_insn_write(void *addr, const void *insn, size_t len)
|
||||
{
|
||||
void *waddr = addr;
|
||||
bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
|
||||
int ret;
|
||||
|
||||
/*
|
||||
* Only two pages can be mapped at a time for writing.
|
||||
*/
|
||||
if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* Before reaching here, it was expected to lock the text_mutex
|
||||
* already, so we don't need to give another lock here and could
|
||||
|
|
@ -74,7 +114,7 @@ static int patch_insn_write(void *addr, const void *insn, size_t len)
|
|||
lockdep_assert_held(&text_mutex);
|
||||
|
||||
if (across_pages)
|
||||
patch_map(addr + len, FIX_TEXT_POKE1);
|
||||
patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
|
||||
|
||||
waddr = patch_map(addr, FIX_TEXT_POKE0);
|
||||
|
||||
|
|
@ -87,15 +127,79 @@ static int patch_insn_write(void *addr, const void *insn, size_t len)
|
|||
|
||||
return ret;
|
||||
}
|
||||
NOKPROBE_SYMBOL(patch_insn_write);
|
||||
NOKPROBE_SYMBOL(__patch_insn_write);
|
||||
#else
|
||||
static int patch_insn_write(void *addr, const void *insn, size_t len)
|
||||
static int __patch_insn_set(void *addr, u8 c, size_t len)
|
||||
{
|
||||
memset(addr, c, len);
|
||||
|
||||
return 0;
|
||||
}
|
||||
NOKPROBE_SYMBOL(__patch_insn_set);
|
||||
|
||||
static int __patch_insn_write(void *addr, const void *insn, size_t len)
|
||||
{
|
||||
return copy_to_kernel_nofault(addr, insn, len);
|
||||
}
|
||||
NOKPROBE_SYMBOL(patch_insn_write);
|
||||
NOKPROBE_SYMBOL(__patch_insn_write);
|
||||
#endif /* CONFIG_MMU */
|
||||
|
||||
static int patch_insn_set(void *addr, u8 c, size_t len)
|
||||
{
|
||||
size_t patched = 0;
|
||||
size_t size;
|
||||
int ret = 0;
|
||||
|
||||
/*
|
||||
* __patch_insn_set() can only work on 2 pages at a time so call it in a
|
||||
* loop with len <= 2 * PAGE_SIZE.
|
||||
*/
|
||||
while (patched < len && !ret) {
|
||||
size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
|
||||
ret = __patch_insn_set(addr + patched, c, size);
|
||||
|
||||
patched += size;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
NOKPROBE_SYMBOL(patch_insn_set);
|
||||
|
||||
int patch_text_set_nosync(void *addr, u8 c, size_t len)
|
||||
{
|
||||
u32 *tp = addr;
|
||||
int ret;
|
||||
|
||||
ret = patch_insn_set(tp, c, len);
|
||||
|
||||
if (!ret)
|
||||
flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len);
|
||||
|
||||
return ret;
|
||||
}
|
||||
NOKPROBE_SYMBOL(patch_text_set_nosync);
|
||||
|
||||
static int patch_insn_write(void *addr, const void *insn, size_t len)
|
||||
{
|
||||
size_t patched = 0;
|
||||
size_t size;
|
||||
int ret = 0;
|
||||
|
||||
/*
|
||||
* Copy the instructions to the destination address, two pages at a time
|
||||
* because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
|
||||
*/
|
||||
while (patched < len && !ret) {
|
||||
size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
|
||||
ret = __patch_insn_write(addr + patched, insn + patched, size);
|
||||
|
||||
patched += size;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
NOKPROBE_SYMBOL(patch_insn_write);
|
||||
|
||||
int patch_text_nosync(void *addr, const void *insns, size_t len)
|
||||
{
|
||||
u32 *tp = addr;
|
||||
|
|
|
|||
|
|
@ -35,5 +35,5 @@ $(obj)/string.o: $(srctree)/lib/string.c FORCE
|
|||
$(obj)/ctype.o: $(srctree)/lib/ctype.c FORCE
|
||||
$(call if_changed_rule,cc_o_c)
|
||||
|
||||
obj-y := cmdline_early.pi.o string.pi.o ctype.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o
|
||||
obj-y := cmdline_early.pi.o fdt_early.pi.o string.pi.o ctype.pi.o lib-fdt.pi.o lib-fdt_ro.pi.o
|
||||
extra-y := $(patsubst %.pi.o,%.o,$(obj-y))
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ static char early_cmdline[COMMAND_LINE_SIZE];
|
|||
* LLVM complain because the function is actually unused in this file).
|
||||
*/
|
||||
u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa);
|
||||
bool set_nokaslr_from_cmdline(uintptr_t dtb_pa);
|
||||
|
||||
static char *get_early_cmdline(uintptr_t dtb_pa)
|
||||
{
|
||||
|
|
@ -60,3 +61,15 @@ u64 set_satp_mode_from_cmdline(uintptr_t dtb_pa)
|
|||
|
||||
return match_noXlvl(cmdline);
|
||||
}
|
||||
|
||||
static bool match_nokaslr(char *cmdline)
|
||||
{
|
||||
return strstr(cmdline, "nokaslr");
|
||||
}
|
||||
|
||||
bool set_nokaslr_from_cmdline(uintptr_t dtb_pa)
|
||||
{
|
||||
char *cmdline = get_early_cmdline(dtb_pa);
|
||||
|
||||
return match_nokaslr(cmdline);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
#include <linux/types.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/libfdt.h>
|
||||
|
||||
/*
|
||||
* Declare the functions that are exported (but prefixed) here so that LLVM
|
||||
* does not complain it lacks the 'static' keyword (which, if added, makes
|
||||
* LLVM complain because the function is actually unused in this file).
|
||||
*/
|
||||
u64 get_kaslr_seed(uintptr_t dtb_pa);
|
||||
|
||||
u64 get_kaslr_seed(uintptr_t dtb_pa)
|
||||
{
|
||||
int node, len;
|
||||
fdt64_t *prop;
|
||||
u64 ret;
|
||||
|
||||
node = fdt_path_offset((void *)dtb_pa, "/chosen");
|
||||
if (node < 0)
|
||||
return 0;
|
||||
|
||||
prop = fdt_getprop_w((void *)dtb_pa, node, "kaslr-seed", &len);
|
||||
if (!prop || len != sizeof(u64))
|
||||
return 0;
|
||||
|
||||
ret = fdt64_to_cpu(*prop);
|
||||
*prop = 0;
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -25,6 +25,9 @@ enum riscv_regset {
|
|||
#ifdef CONFIG_FPU
|
||||
REGSET_F,
|
||||
#endif
|
||||
#ifdef CONFIG_RISCV_ISA_V
|
||||
REGSET_V,
|
||||
#endif
|
||||
};
|
||||
|
||||
static int riscv_gpr_get(struct task_struct *target,
|
||||
|
|
@ -81,6 +84,71 @@ static int riscv_fpr_set(struct task_struct *target,
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_RISCV_ISA_V
|
||||
static int riscv_vr_get(struct task_struct *target,
|
||||
const struct user_regset *regset,
|
||||
struct membuf to)
|
||||
{
|
||||
struct __riscv_v_ext_state *vstate = &target->thread.vstate;
|
||||
struct __riscv_v_regset_state ptrace_vstate;
|
||||
|
||||
if (!riscv_v_vstate_query(task_pt_regs(target)))
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* Ensure the vector registers have been saved to the memory before
|
||||
* copying them to membuf.
|
||||
*/
|
||||
if (target == current)
|
||||
riscv_v_vstate_save(current, task_pt_regs(current));
|
||||
|
||||
ptrace_vstate.vstart = vstate->vstart;
|
||||
ptrace_vstate.vl = vstate->vl;
|
||||
ptrace_vstate.vtype = vstate->vtype;
|
||||
ptrace_vstate.vcsr = vstate->vcsr;
|
||||
ptrace_vstate.vlenb = vstate->vlenb;
|
||||
|
||||
/* Copy vector header from vstate. */
|
||||
membuf_write(&to, &ptrace_vstate, sizeof(struct __riscv_v_regset_state));
|
||||
|
||||
/* Copy all the vector registers from vstate. */
|
||||
return membuf_write(&to, vstate->datap, riscv_v_vsize);
|
||||
}
|
||||
|
||||
static int riscv_vr_set(struct task_struct *target,
|
||||
const struct user_regset *regset,
|
||||
unsigned int pos, unsigned int count,
|
||||
const void *kbuf, const void __user *ubuf)
|
||||
{
|
||||
int ret;
|
||||
struct __riscv_v_ext_state *vstate = &target->thread.vstate;
|
||||
struct __riscv_v_regset_state ptrace_vstate;
|
||||
|
||||
if (!riscv_v_vstate_query(task_pt_regs(target)))
|
||||
return -EINVAL;
|
||||
|
||||
/* Copy rest of the vstate except datap */
|
||||
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ptrace_vstate, 0,
|
||||
sizeof(struct __riscv_v_regset_state));
|
||||
if (unlikely(ret))
|
||||
return ret;
|
||||
|
||||
if (vstate->vlenb != ptrace_vstate.vlenb)
|
||||
return -EINVAL;
|
||||
|
||||
vstate->vstart = ptrace_vstate.vstart;
|
||||
vstate->vl = ptrace_vstate.vl;
|
||||
vstate->vtype = ptrace_vstate.vtype;
|
||||
vstate->vcsr = ptrace_vstate.vcsr;
|
||||
|
||||
/* Copy all the vector registers. */
|
||||
pos = 0;
|
||||
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate->datap,
|
||||
0, riscv_v_vsize);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
static const struct user_regset riscv_user_regset[] = {
|
||||
[REGSET_X] = {
|
||||
.core_note_type = NT_PRSTATUS,
|
||||
|
|
@ -100,6 +168,17 @@ static const struct user_regset riscv_user_regset[] = {
|
|||
.set = riscv_fpr_set,
|
||||
},
|
||||
#endif
|
||||
#ifdef CONFIG_RISCV_ISA_V
|
||||
[REGSET_V] = {
|
||||
.core_note_type = NT_RISCV_VECTOR,
|
||||
.align = 16,
|
||||
.n = ((32 * RISCV_MAX_VLENB) +
|
||||
sizeof(struct __riscv_v_regset_state)) / sizeof(__u32),
|
||||
.size = sizeof(__u32),
|
||||
.regset_get = riscv_vr_get,
|
||||
.set = riscv_vr_set,
|
||||
},
|
||||
#endif
|
||||
};
|
||||
|
||||
static const struct user_regset_view riscv_user_native_view = {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
#include <linux/smp.h>
|
||||
#include <linux/efi.h>
|
||||
#include <linux/crash_dump.h>
|
||||
#include <linux/panic_notifier.h>
|
||||
|
||||
#include <asm/acpi.h>
|
||||
#include <asm/alternative.h>
|
||||
|
|
@ -347,3 +348,27 @@ void free_initmem(void)
|
|||
|
||||
free_initmem_default(POISON_FREE_INITMEM);
|
||||
}
|
||||
|
||||
static int dump_kernel_offset(struct notifier_block *self,
|
||||
unsigned long v, void *p)
|
||||
{
|
||||
pr_emerg("Kernel Offset: 0x%lx from 0x%lx\n",
|
||||
kernel_map.virt_offset,
|
||||
KERNEL_LINK_ADDR);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct notifier_block kernel_offset_notifier = {
|
||||
.notifier_call = dump_kernel_offset
|
||||
};
|
||||
|
||||
static int __init register_kernel_offset_dumper(void)
|
||||
{
|
||||
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
|
||||
atomic_notifier_chain_register(&panic_notifier_list,
|
||||
&kernel_offset_notifier);
|
||||
|
||||
return 0;
|
||||
}
|
||||
device_initcall(register_kernel_offset_dumper);
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
#include <linux/sched/task_stack.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <asm/cpu_ops.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/irq.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/numa.h>
|
||||
|
|
@ -245,7 +246,7 @@ asmlinkage __visible void smp_callin(void)
|
|||
|
||||
numa_add_cpu(curr_cpuid);
|
||||
set_cpu_online(curr_cpuid, 1);
|
||||
probe_vendor_features(curr_cpuid);
|
||||
check_unaligned_access(curr_cpuid);
|
||||
|
||||
if (has_vector()) {
|
||||
if (riscv_v_setup_vsize())
|
||||
|
|
|
|||
|
|
@ -9,26 +9,93 @@
|
|||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/mm.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/dma-noncoherent.h>
|
||||
|
||||
static bool noncoherent_supported __ro_after_init;
|
||||
int dma_cache_alignment __ro_after_init = ARCH_DMA_MINALIGN;
|
||||
EXPORT_SYMBOL_GPL(dma_cache_alignment);
|
||||
|
||||
void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
|
||||
enum dma_data_direction dir)
|
||||
struct riscv_nonstd_cache_ops noncoherent_cache_ops __ro_after_init = {
|
||||
.wback = NULL,
|
||||
.inv = NULL,
|
||||
.wback_inv = NULL,
|
||||
};
|
||||
|
||||
static inline void arch_dma_cache_wback(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
void *vaddr = phys_to_virt(paddr);
|
||||
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.wback)) {
|
||||
noncoherent_cache_ops.wback(paddr, size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ALT_CMO_OP(clean, vaddr, size, riscv_cbom_block_size);
|
||||
}
|
||||
|
||||
static inline void arch_dma_cache_inv(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
void *vaddr = phys_to_virt(paddr);
|
||||
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.inv)) {
|
||||
noncoherent_cache_ops.inv(paddr, size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
ALT_CMO_OP(inval, vaddr, size, riscv_cbom_block_size);
|
||||
}
|
||||
|
||||
static inline void arch_dma_cache_wback_inv(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
void *vaddr = phys_to_virt(paddr);
|
||||
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.wback_inv)) {
|
||||
noncoherent_cache_ops.wback_inv(paddr, size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
ALT_CMO_OP(flush, vaddr, size, riscv_cbom_block_size);
|
||||
}
|
||||
|
||||
static inline bool arch_sync_dma_clean_before_fromdevice(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool arch_sync_dma_cpu_needs_post_dma_flush(void)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
switch (dir) {
|
||||
case DMA_TO_DEVICE:
|
||||
ALT_CMO_OP(clean, vaddr, size, riscv_cbom_block_size);
|
||||
arch_dma_cache_wback(paddr, size);
|
||||
break;
|
||||
|
||||
case DMA_FROM_DEVICE:
|
||||
ALT_CMO_OP(clean, vaddr, size, riscv_cbom_block_size);
|
||||
break;
|
||||
if (!arch_sync_dma_clean_before_fromdevice()) {
|
||||
arch_dma_cache_inv(paddr, size);
|
||||
break;
|
||||
}
|
||||
fallthrough;
|
||||
|
||||
case DMA_BIDIRECTIONAL:
|
||||
ALT_CMO_OP(flush, vaddr, size, riscv_cbom_block_size);
|
||||
/* Skip the invalidate here if it's done later */
|
||||
if (IS_ENABLED(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) &&
|
||||
arch_sync_dma_cpu_needs_post_dma_flush())
|
||||
arch_dma_cache_wback(paddr, size);
|
||||
else
|
||||
arch_dma_cache_wback_inv(paddr, size);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -37,15 +104,17 @@ void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
|
|||
void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
void *vaddr = phys_to_virt(paddr);
|
||||
|
||||
switch (dir) {
|
||||
case DMA_TO_DEVICE:
|
||||
break;
|
||||
|
||||
case DMA_FROM_DEVICE:
|
||||
case DMA_BIDIRECTIONAL:
|
||||
ALT_CMO_OP(flush, vaddr, size, riscv_cbom_block_size);
|
||||
/* FROM_DEVICE invalidate needed if speculative CPU prefetch only */
|
||||
if (arch_sync_dma_cpu_needs_post_dma_flush())
|
||||
arch_dma_cache_inv(paddr, size);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -55,6 +124,13 @@ void arch_dma_prep_coherent(struct page *page, size_t size)
|
|||
{
|
||||
void *flush_addr = page_address(page);
|
||||
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.wback_inv)) {
|
||||
noncoherent_cache_ops.wback_inv(page_to_phys(page), size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
ALT_CMO_OP(flush, flush_addr, size, riscv_cbom_block_size);
|
||||
}
|
||||
|
||||
|
|
@ -86,3 +162,12 @@ void __init riscv_set_dma_cache_alignment(void)
|
|||
if (!noncoherent_supported)
|
||||
dma_cache_alignment = 1;
|
||||
}
|
||||
|
||||
void riscv_noncoherent_register_cache_ops(const struct riscv_nonstd_cache_ops *ops)
|
||||
{
|
||||
if (!ops)
|
||||
return;
|
||||
|
||||
noncoherent_cache_ops = *ops;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(riscv_noncoherent_register_cache_ops);
|
||||
|
|
|
|||
|
|
@ -1014,11 +1014,45 @@ static void __init pt_ops_set_late(void)
|
|||
#endif
|
||||
}
|
||||
|
||||
#ifdef CONFIG_RANDOMIZE_BASE
|
||||
extern bool __init __pi_set_nokaslr_from_cmdline(uintptr_t dtb_pa);
|
||||
extern u64 __init __pi_get_kaslr_seed(uintptr_t dtb_pa);
|
||||
|
||||
static int __init print_nokaslr(char *p)
|
||||
{
|
||||
pr_info("Disabled KASLR");
|
||||
return 0;
|
||||
}
|
||||
early_param("nokaslr", print_nokaslr);
|
||||
|
||||
unsigned long kaslr_offset(void)
|
||||
{
|
||||
return kernel_map.virt_offset;
|
||||
}
|
||||
#endif
|
||||
|
||||
asmlinkage void __init setup_vm(uintptr_t dtb_pa)
|
||||
{
|
||||
pmd_t __maybe_unused fix_bmap_spmd, fix_bmap_epmd;
|
||||
|
||||
kernel_map.virt_addr = KERNEL_LINK_ADDR;
|
||||
#ifdef CONFIG_RANDOMIZE_BASE
|
||||
if (!__pi_set_nokaslr_from_cmdline(dtb_pa)) {
|
||||
u64 kaslr_seed = __pi_get_kaslr_seed(dtb_pa);
|
||||
u32 kernel_size = (uintptr_t)(&_end) - (uintptr_t)(&_start);
|
||||
u32 nr_pos;
|
||||
|
||||
/*
|
||||
* Compute the number of positions available: we are limited
|
||||
* by the early page table that only has one PUD and we must
|
||||
* be aligned on PMD_SIZE.
|
||||
*/
|
||||
nr_pos = (PUD_SIZE - kernel_size) / PMD_SIZE;
|
||||
|
||||
kernel_map.virt_offset = (kaslr_seed % nr_pos) * PMD_SIZE;
|
||||
}
|
||||
#endif
|
||||
|
||||
kernel_map.virt_addr = KERNEL_LINK_ADDR + kernel_map.virt_offset;
|
||||
kernel_map.page_offset = _AC(CONFIG_PAGE_OFFSET, UL);
|
||||
|
||||
#ifdef CONFIG_XIP_KERNEL
|
||||
|
|
|
|||
|
|
@ -7,15 +7,28 @@
|
|||
#include <linux/libnvdimm.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/dma-noncoherent.h>
|
||||
|
||||
void arch_wb_cache_pmem(void *addr, size_t size)
|
||||
{
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.wback)) {
|
||||
noncoherent_cache_ops.wback(virt_to_phys(addr), size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ALT_CMO_OP(clean, addr, size, riscv_cbom_block_size);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
|
||||
|
||||
void arch_invalidate_pmem(void *addr, size_t size)
|
||||
{
|
||||
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
|
||||
if (unlikely(noncoherent_cache_ops.inv)) {
|
||||
noncoherent_cache_ops.inv(virt_to_phys(addr), size);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ALT_CMO_OP(inval, addr, size, riscv_cbom_block_size);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ static inline bool is_creg(u8 reg)
|
|||
struct rv_jit_context {
|
||||
struct bpf_prog *prog;
|
||||
u16 *insns; /* RV insns */
|
||||
u16 *ro_insns;
|
||||
int ninsns;
|
||||
int prologue_len;
|
||||
int epilogue_offset;
|
||||
|
|
@ -85,7 +86,9 @@ static inline int ninsns_rvoff(int ninsns)
|
|||
|
||||
struct rv_jit_data {
|
||||
struct bpf_binary_header *header;
|
||||
struct bpf_binary_header *ro_header;
|
||||
u8 *image;
|
||||
u8 *ro_image;
|
||||
struct rv_jit_context ctx;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -144,7 +144,11 @@ static bool in_auipc_jalr_range(s64 val)
|
|||
/* Emit fixed-length instructions for address */
|
||||
static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx)
|
||||
{
|
||||
u64 ip = (u64)(ctx->insns + ctx->ninsns);
|
||||
/*
|
||||
* Use the ro_insns(RX) to calculate the offset as the BPF program will
|
||||
* finally run from this memory region.
|
||||
*/
|
||||
u64 ip = (u64)(ctx->ro_insns + ctx->ninsns);
|
||||
s64 off = addr - ip;
|
||||
s64 upper = (off + (1 << 11)) >> 12;
|
||||
s64 lower = off & 0xfff;
|
||||
|
|
@ -464,8 +468,12 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx)
|
|||
s64 off = 0;
|
||||
u64 ip;
|
||||
|
||||
if (addr && ctx->insns) {
|
||||
ip = (u64)(long)(ctx->insns + ctx->ninsns);
|
||||
if (addr && ctx->insns && ctx->ro_insns) {
|
||||
/*
|
||||
* Use the ro_insns(RX) to calculate the offset as the BPF
|
||||
* program will finally run from this memory region.
|
||||
*/
|
||||
ip = (u64)(long)(ctx->ro_insns + ctx->ninsns);
|
||||
off = addr - ip;
|
||||
}
|
||||
|
||||
|
|
@ -578,9 +586,10 @@ static int add_exception_handler(const struct bpf_insn *insn,
|
|||
{
|
||||
struct exception_table_entry *ex;
|
||||
unsigned long pc;
|
||||
off_t offset;
|
||||
off_t ins_offset;
|
||||
off_t fixup_offset;
|
||||
|
||||
if (!ctx->insns || !ctx->prog->aux->extable ||
|
||||
if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable ||
|
||||
(BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))
|
||||
return 0;
|
||||
|
||||
|
|
@ -594,12 +603,17 @@ static int add_exception_handler(const struct bpf_insn *insn,
|
|||
return -EINVAL;
|
||||
|
||||
ex = &ctx->prog->aux->extable[ctx->nexentries];
|
||||
pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len];
|
||||
pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len];
|
||||
|
||||
offset = pc - (long)&ex->insn;
|
||||
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
|
||||
/*
|
||||
* This is the relative offset of the instruction that may fault from
|
||||
* the exception table itself. This will be written to the exception
|
||||
* table and if this instruction faults, the destination register will
|
||||
* be set to '0' and the execution will jump to the next instruction.
|
||||
*/
|
||||
ins_offset = pc - (long)&ex->insn;
|
||||
if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
|
||||
return -ERANGE;
|
||||
ex->insn = offset;
|
||||
|
||||
/*
|
||||
* Since the extable follows the program, the fixup offset is always
|
||||
|
|
@ -608,12 +622,25 @@ static int add_exception_handler(const struct bpf_insn *insn,
|
|||
* bits. We don't need to worry about buildtime or runtime sort
|
||||
* modifying the upper bits because the table is already sorted, and
|
||||
* isn't part of the main exception table.
|
||||
*
|
||||
* The fixup_offset is set to the next instruction from the instruction
|
||||
* that may fault. The execution will jump to this after handling the
|
||||
* fault.
|
||||
*/
|
||||
offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
|
||||
if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
|
||||
fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
|
||||
if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
|
||||
return -ERANGE;
|
||||
|
||||
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
|
||||
/*
|
||||
* The offsets above have been calculated using the RO buffer but we
|
||||
* need to use the R/W buffer for writes.
|
||||
* switch ex to rw buffer for writing.
|
||||
*/
|
||||
ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns);
|
||||
|
||||
ex->insn = ins_offset;
|
||||
|
||||
ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
|
||||
FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
|
||||
ex->type = EX_TYPE_BPF;
|
||||
|
||||
|
|
@ -1007,6 +1034,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
|
||||
ctx.ninsns = 0;
|
||||
ctx.insns = NULL;
|
||||
ctx.ro_insns = NULL;
|
||||
ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
|
@ -1015,7 +1043,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
|
|||
return -EFBIG;
|
||||
|
||||
ctx.ninsns = 0;
|
||||
/*
|
||||
* The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the
|
||||
* JITed instructions and later copies it to a RX region (ctx.ro_insns).
|
||||
* It also uses ctx.ro_insns to calculate offsets for jumps etc. As the
|
||||
* trampoline image uses the same memory area for writing and execution,
|
||||
* both ctx.insns and ctx.ro_insns can be set to image.
|
||||
*/
|
||||
ctx.insns = image;
|
||||
ctx.ro_insns = image;
|
||||
ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
|
||||
#include <linux/bpf.h>
|
||||
#include <linux/filter.h>
|
||||
#include <linux/memory.h>
|
||||
#include <asm/patch.h>
|
||||
#include "bpf_jit.h"
|
||||
|
||||
/* Number of iterations to try until offsets converge. */
|
||||
|
|
@ -117,16 +119,24 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
sizeof(struct exception_table_entry);
|
||||
prog_size = sizeof(*ctx->insns) * ctx->ninsns;
|
||||
|
||||
jit_data->header =
|
||||
bpf_jit_binary_alloc(prog_size + extable_size,
|
||||
&jit_data->image,
|
||||
sizeof(u32),
|
||||
bpf_fill_ill_insns);
|
||||
if (!jit_data->header) {
|
||||
jit_data->ro_header =
|
||||
bpf_jit_binary_pack_alloc(prog_size + extable_size,
|
||||
&jit_data->ro_image, sizeof(u32),
|
||||
&jit_data->header, &jit_data->image,
|
||||
bpf_fill_ill_insns);
|
||||
if (!jit_data->ro_header) {
|
||||
prog = orig_prog;
|
||||
goto out_offset;
|
||||
}
|
||||
|
||||
/*
|
||||
* Use the image(RW) for writing the JITed instructions. But also save
|
||||
* the ro_image(RX) for calculating the offsets in the image. The RW
|
||||
* image will be later copied to the RX image from where the program
|
||||
* will run. The bpf_jit_binary_pack_finalize() will do this copy in the
|
||||
* final step.
|
||||
*/
|
||||
ctx->ro_insns = (u16 *)jit_data->ro_image;
|
||||
ctx->insns = (u16 *)jit_data->image;
|
||||
/*
|
||||
* Now, when the image is allocated, the image can
|
||||
|
|
@ -138,14 +148,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
|
||||
if (i == NR_JIT_ITERATIONS) {
|
||||
pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
|
||||
if (jit_data->header)
|
||||
bpf_jit_binary_free(jit_data->header);
|
||||
prog = orig_prog;
|
||||
goto out_offset;
|
||||
goto out_free_hdr;
|
||||
}
|
||||
|
||||
if (extable_size)
|
||||
prog->aux->extable = (void *)ctx->insns + prog_size;
|
||||
prog->aux->extable = (void *)ctx->ro_insns + prog_size;
|
||||
|
||||
skip_init_ctx:
|
||||
pass++;
|
||||
|
|
@ -154,23 +162,33 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
|
||||
bpf_jit_build_prologue(ctx);
|
||||
if (build_body(ctx, extra_pass, NULL)) {
|
||||
bpf_jit_binary_free(jit_data->header);
|
||||
prog = orig_prog;
|
||||
goto out_offset;
|
||||
goto out_free_hdr;
|
||||
}
|
||||
bpf_jit_build_epilogue(ctx);
|
||||
|
||||
if (bpf_jit_enable > 1)
|
||||
bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
|
||||
|
||||
prog->bpf_func = (void *)ctx->insns;
|
||||
prog->bpf_func = (void *)ctx->ro_insns;
|
||||
prog->jited = 1;
|
||||
prog->jited_len = prog_size;
|
||||
|
||||
bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
|
||||
|
||||
if (!prog->is_func || extra_pass) {
|
||||
bpf_jit_binary_lock_ro(jit_data->header);
|
||||
if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
|
||||
jit_data->header))) {
|
||||
/* ro_header has been freed */
|
||||
jit_data->ro_header = NULL;
|
||||
prog = orig_prog;
|
||||
goto out_offset;
|
||||
}
|
||||
/*
|
||||
* The instructions have now been copied to the ROX region from
|
||||
* where they will execute.
|
||||
* Write any modified data cache blocks out to memory and
|
||||
* invalidate the corresponding blocks in the instruction cache.
|
||||
*/
|
||||
bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
|
||||
for (i = 0; i < prog->len; i++)
|
||||
ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
|
||||
bpf_prog_fill_jited_linfo(prog, ctx->offset);
|
||||
|
|
@ -185,6 +203,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
bpf_jit_prog_release_other(prog, prog == orig_prog ?
|
||||
tmp : orig_prog);
|
||||
return prog;
|
||||
|
||||
out_free_hdr:
|
||||
if (jit_data->header) {
|
||||
bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
|
||||
sizeof(jit_data->header->size));
|
||||
bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
|
||||
}
|
||||
goto out_offset;
|
||||
}
|
||||
|
||||
u64 bpf_jit_alloc_exec_limit(void)
|
||||
|
|
@ -204,3 +230,51 @@ void bpf_jit_free_exec(void *addr)
|
|||
{
|
||||
return vfree(addr);
|
||||
}
|
||||
|
||||
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&text_mutex);
|
||||
ret = patch_text_nosync(dst, src, len);
|
||||
mutex_unlock(&text_mutex);
|
||||
|
||||
if (ret)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
return dst;
|
||||
}
|
||||
|
||||
int bpf_arch_text_invalidate(void *dst, size_t len)
|
||||
{
|
||||
int ret;
|
||||
|
||||
mutex_lock(&text_mutex);
|
||||
ret = patch_text_set_nosync(dst, 0, len);
|
||||
mutex_unlock(&text_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void bpf_jit_free(struct bpf_prog *prog)
|
||||
{
|
||||
if (prog->jited) {
|
||||
struct rv_jit_data *jit_data = prog->aux->jit_data;
|
||||
struct bpf_binary_header *hdr;
|
||||
|
||||
/*
|
||||
* If we fail the final pass of JIT (from jit_subprogs),
|
||||
* the program may not be finalized yet. Call finalize here
|
||||
* before freeing it.
|
||||
*/
|
||||
if (jit_data) {
|
||||
bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
|
||||
kfree(jit_data);
|
||||
}
|
||||
hdr = bpf_jit_binary_pack_hdr(prog);
|
||||
bpf_jit_binary_pack_free(hdr, NULL);
|
||||
WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
|
||||
}
|
||||
|
||||
bpf_prog_unlock_free(prog);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ source "drivers/base/Kconfig"
|
|||
|
||||
source "drivers/bus/Kconfig"
|
||||
|
||||
source "drivers/cache/Kconfig"
|
||||
|
||||
source "drivers/connector/Kconfig"
|
||||
|
||||
source "drivers/firmware/Kconfig"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ ifdef building_out_of_srctree
|
|||
MAKEFLAGS += --include-dir=$(srctree)
|
||||
endif
|
||||
|
||||
obj-y += cache/
|
||||
obj-y += irqchip/
|
||||
obj-y += bus/
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
# SPDX-License-Identifier: GPL-2.0
|
||||
menu "Cache Drivers"
|
||||
|
||||
config AX45MP_L2_CACHE
|
||||
bool "Andes Technology AX45MP L2 Cache controller"
|
||||
depends on RISCV_DMA_NONCOHERENT
|
||||
select RISCV_NONSTANDARD_CACHE_OPS
|
||||
help
|
||||
Support for the L2 cache controller on Andes Technology AX45MP platforms.
|
||||
|
||||
endmenu
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
obj-$(CONFIG_AX45MP_L2_CACHE) += ax45mp_cache.o
|
||||
|
|
@ -0,0 +1,213 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* non-coherent cache functions for Andes AX45MP
|
||||
*
|
||||
* Copyright (C) 2023 Renesas Electronics Corp.
|
||||
*/
|
||||
|
||||
#include <linux/cacheflush.h>
|
||||
#include <linux/cacheinfo.h>
|
||||
#include <linux/dma-direction.h>
|
||||
#include <linux/of_address.h>
|
||||
#include <linux/of_platform.h>
|
||||
|
||||
#include <asm/dma-noncoherent.h>
|
||||
|
||||
/* L2 cache registers */
|
||||
#define AX45MP_L2C_REG_CTL_OFFSET 0x8
|
||||
|
||||
#define AX45MP_L2C_REG_C0_CMD_OFFSET 0x40
|
||||
#define AX45MP_L2C_REG_C0_ACC_OFFSET 0x48
|
||||
#define AX45MP_L2C_REG_STATUS_OFFSET 0x80
|
||||
|
||||
/* D-cache operation */
|
||||
#define AX45MP_CCTL_L1D_VA_INVAL 0 /* Invalidate an L1 cache entry */
|
||||
#define AX45MP_CCTL_L1D_VA_WB 1 /* Write-back an L1 cache entry */
|
||||
|
||||
/* L2 CCTL status */
|
||||
#define AX45MP_CCTL_L2_STATUS_IDLE 0
|
||||
|
||||
/* L2 CCTL status cores mask */
|
||||
#define AX45MP_CCTL_L2_STATUS_C0_MASK 0xf
|
||||
|
||||
/* L2 cache operation */
|
||||
#define AX45MP_CCTL_L2_PA_INVAL 0x8 /* Invalidate an L2 cache entry */
|
||||
#define AX45MP_CCTL_L2_PA_WB 0x9 /* Write-back an L2 cache entry */
|
||||
|
||||
#define AX45MP_L2C_REG_PER_CORE_OFFSET 0x10
|
||||
#define AX45MP_CCTL_L2_STATUS_PER_CORE_OFFSET 4
|
||||
|
||||
#define AX45MP_L2C_REG_CN_CMD_OFFSET(n) \
|
||||
(AX45MP_L2C_REG_C0_CMD_OFFSET + ((n) * AX45MP_L2C_REG_PER_CORE_OFFSET))
|
||||
#define AX45MP_L2C_REG_CN_ACC_OFFSET(n) \
|
||||
(AX45MP_L2C_REG_C0_ACC_OFFSET + ((n) * AX45MP_L2C_REG_PER_CORE_OFFSET))
|
||||
#define AX45MP_CCTL_L2_STATUS_CN_MASK(n) \
|
||||
(AX45MP_CCTL_L2_STATUS_C0_MASK << ((n) * AX45MP_CCTL_L2_STATUS_PER_CORE_OFFSET))
|
||||
|
||||
#define AX45MP_CCTL_REG_UCCTLBEGINADDR_NUM 0x80b
|
||||
#define AX45MP_CCTL_REG_UCCTLCOMMAND_NUM 0x80c
|
||||
|
||||
#define AX45MP_CACHE_LINE_SIZE 64
|
||||
|
||||
struct ax45mp_priv {
|
||||
void __iomem *l2c_base;
|
||||
u32 ax45mp_cache_line_size;
|
||||
};
|
||||
|
||||
static struct ax45mp_priv ax45mp_priv;
|
||||
|
||||
/* L2 Cache operations */
|
||||
static inline uint32_t ax45mp_cpu_l2c_get_cctl_status(void)
|
||||
{
|
||||
return readl(ax45mp_priv.l2c_base + AX45MP_L2C_REG_STATUS_OFFSET);
|
||||
}
|
||||
|
||||
static void ax45mp_cpu_cache_operation(unsigned long start, unsigned long end,
|
||||
unsigned int l1_op, unsigned int l2_op)
|
||||
{
|
||||
unsigned long line_size = ax45mp_priv.ax45mp_cache_line_size;
|
||||
void __iomem *base = ax45mp_priv.l2c_base;
|
||||
int mhartid = smp_processor_id();
|
||||
unsigned long pa;
|
||||
|
||||
while (end > start) {
|
||||
csr_write(AX45MP_CCTL_REG_UCCTLBEGINADDR_NUM, start);
|
||||
csr_write(AX45MP_CCTL_REG_UCCTLCOMMAND_NUM, l1_op);
|
||||
|
||||
pa = virt_to_phys((void *)start);
|
||||
writel(pa, base + AX45MP_L2C_REG_CN_ACC_OFFSET(mhartid));
|
||||
writel(l2_op, base + AX45MP_L2C_REG_CN_CMD_OFFSET(mhartid));
|
||||
while ((ax45mp_cpu_l2c_get_cctl_status() &
|
||||
AX45MP_CCTL_L2_STATUS_CN_MASK(mhartid)) !=
|
||||
AX45MP_CCTL_L2_STATUS_IDLE)
|
||||
;
|
||||
|
||||
start += line_size;
|
||||
}
|
||||
}
|
||||
|
||||
/* Write-back L1 and L2 cache entry */
|
||||
static inline void ax45mp_cpu_dcache_wb_range(unsigned long start, unsigned long end)
|
||||
{
|
||||
ax45mp_cpu_cache_operation(start, end, AX45MP_CCTL_L1D_VA_WB,
|
||||
AX45MP_CCTL_L2_PA_WB);
|
||||
}
|
||||
|
||||
/* Invalidate the L1 and L2 cache entry */
|
||||
static inline void ax45mp_cpu_dcache_inval_range(unsigned long start, unsigned long end)
|
||||
{
|
||||
ax45mp_cpu_cache_operation(start, end, AX45MP_CCTL_L1D_VA_INVAL,
|
||||
AX45MP_CCTL_L2_PA_INVAL);
|
||||
}
|
||||
|
||||
static void ax45mp_dma_cache_inv(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
unsigned long start = (unsigned long)phys_to_virt(paddr);
|
||||
unsigned long end = start + size;
|
||||
unsigned long line_size;
|
||||
unsigned long flags;
|
||||
|
||||
if (unlikely(start == end))
|
||||
return;
|
||||
|
||||
line_size = ax45mp_priv.ax45mp_cache_line_size;
|
||||
|
||||
start = start & (~(line_size - 1));
|
||||
end = ((end + line_size - 1) & (~(line_size - 1)));
|
||||
|
||||
local_irq_save(flags);
|
||||
|
||||
ax45mp_cpu_dcache_inval_range(start, end);
|
||||
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
||||
static void ax45mp_dma_cache_wback(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
unsigned long start = (unsigned long)phys_to_virt(paddr);
|
||||
unsigned long end = start + size;
|
||||
unsigned long line_size;
|
||||
unsigned long flags;
|
||||
|
||||
line_size = ax45mp_priv.ax45mp_cache_line_size;
|
||||
start = start & (~(line_size - 1));
|
||||
local_irq_save(flags);
|
||||
ax45mp_cpu_dcache_wb_range(start, end);
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
||||
static void ax45mp_dma_cache_wback_inv(phys_addr_t paddr, size_t size)
|
||||
{
|
||||
ax45mp_dma_cache_wback(paddr, size);
|
||||
ax45mp_dma_cache_inv(paddr, size);
|
||||
}
|
||||
|
||||
static int ax45mp_get_l2_line_size(struct device_node *np)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = of_property_read_u32(np, "cache-line-size", &ax45mp_priv.ax45mp_cache_line_size);
|
||||
if (ret) {
|
||||
pr_err("Failed to get cache-line-size, defaulting to 64 bytes\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (ax45mp_priv.ax45mp_cache_line_size != AX45MP_CACHE_LINE_SIZE) {
|
||||
pr_err("Expected cache-line-size to be 64 bytes (found:%u)\n",
|
||||
ax45mp_priv.ax45mp_cache_line_size);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct riscv_nonstd_cache_ops ax45mp_cmo_ops __initdata = {
|
||||
.wback = &ax45mp_dma_cache_wback,
|
||||
.inv = &ax45mp_dma_cache_inv,
|
||||
.wback_inv = &ax45mp_dma_cache_wback_inv,
|
||||
};
|
||||
|
||||
static const struct of_device_id ax45mp_cache_ids[] = {
|
||||
{ .compatible = "andestech,ax45mp-cache" },
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
|
||||
static int __init ax45mp_cache_init(void)
|
||||
{
|
||||
struct device_node *np;
|
||||
struct resource res;
|
||||
int ret;
|
||||
|
||||
np = of_find_matching_node(NULL, ax45mp_cache_ids);
|
||||
if (!of_device_is_available(np))
|
||||
return -ENODEV;
|
||||
|
||||
ret = of_address_to_resource(np, 0, &res);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/*
|
||||
* If IOCP is present on the Andes AX45MP core riscv_cbom_block_size
|
||||
* will be 0 for sure, so we can definitely rely on it. If
|
||||
* riscv_cbom_block_size = 0 we don't need to handle CMO using SW any
|
||||
* more so we just return success here and only if its being set we
|
||||
* continue further in the probe path.
|
||||
*/
|
||||
if (!riscv_cbom_block_size)
|
||||
return 0;
|
||||
|
||||
ax45mp_priv.l2c_base = ioremap(res.start, resource_size(&res));
|
||||
if (!ax45mp_priv.l2c_base)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = ax45mp_get_l2_line_size(np);
|
||||
if (ret) {
|
||||
iounmap(ax45mp_priv.l2c_base);
|
||||
return ret;
|
||||
}
|
||||
|
||||
riscv_noncoherent_register_cache_ops(&ax45mp_cmo_ops);
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_initcall(ax45mp_cache_init);
|
||||
|
|
@ -86,10 +86,10 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \
|
|||
screen_info.o efi-stub-entry.o
|
||||
|
||||
lib-$(CONFIG_ARM) += arm32-stub.o
|
||||
lib-$(CONFIG_ARM64) += arm64.o arm64-stub.o smbios.o
|
||||
lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o
|
||||
lib-$(CONFIG_X86) += x86-stub.o
|
||||
lib-$(CONFIG_X86_64) += x86-5lvl.o
|
||||
lib-$(CONFIG_RISCV) += riscv.o riscv-stub.o
|
||||
lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o
|
||||
lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o
|
||||
|
||||
CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
|
||||
|
|
|
|||
|
|
@ -14,42 +14,6 @@
|
|||
|
||||
#include "efistub.h"
|
||||
|
||||
/*
|
||||
* Distro versions of GRUB may ignore the BSS allocation entirely (i.e., fail
|
||||
* to provide space, and fail to zero it). Check for this condition by double
|
||||
* checking that the first and the last byte of the image are covered by the
|
||||
* same EFI memory map entry.
|
||||
*/
|
||||
static bool check_image_region(u64 base, u64 size)
|
||||
{
|
||||
struct efi_boot_memmap *map;
|
||||
efi_status_t status;
|
||||
bool ret = false;
|
||||
int map_offset;
|
||||
|
||||
status = efi_get_memory_map(&map, false);
|
||||
if (status != EFI_SUCCESS)
|
||||
return false;
|
||||
|
||||
for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) {
|
||||
efi_memory_desc_t *md = (void *)map->map + map_offset;
|
||||
u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE;
|
||||
|
||||
/*
|
||||
* Find the region that covers base, and return whether
|
||||
* it covers base+size bytes.
|
||||
*/
|
||||
if (base >= md->phys_addr && base < end) {
|
||||
ret = (base + size) <= end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
efi_bs_call(free_pool, map);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
efi_status_t handle_kernel_image(unsigned long *image_addr,
|
||||
unsigned long *image_size,
|
||||
unsigned long *reserve_addr,
|
||||
|
|
@ -59,31 +23,6 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
|
|||
{
|
||||
efi_status_t status;
|
||||
unsigned long kernel_size, kernel_codesize, kernel_memsize;
|
||||
u32 phys_seed = 0;
|
||||
u64 min_kimg_align = efi_get_kimg_min_align();
|
||||
|
||||
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
|
||||
efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID;
|
||||
void *p;
|
||||
|
||||
if (efi_nokaslr) {
|
||||
efi_info("KASLR disabled on kernel command line\n");
|
||||
} else if (efi_bs_call(handle_protocol, image_handle,
|
||||
&li_fixed_proto, &p) == EFI_SUCCESS) {
|
||||
efi_info("Image placement fixed by loader\n");
|
||||
} else {
|
||||
status = efi_get_random_bytes(sizeof(phys_seed),
|
||||
(u8 *)&phys_seed);
|
||||
if (status == EFI_NOT_FOUND) {
|
||||
efi_info("EFI_RNG_PROTOCOL unavailable\n");
|
||||
efi_nokaslr = true;
|
||||
} else if (status != EFI_SUCCESS) {
|
||||
efi_err("efi_get_random_bytes() failed (0x%lx)\n",
|
||||
status);
|
||||
efi_nokaslr = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (image->image_base != _text) {
|
||||
efi_err("FIRMWARE BUG: efi_loaded_image_t::image_base has bogus value\n");
|
||||
|
|
@ -98,50 +37,15 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
|
|||
kernel_codesize = __inittext_end - _text;
|
||||
kernel_memsize = kernel_size + (_end - _edata);
|
||||
*reserve_size = kernel_memsize;
|
||||
*image_addr = (unsigned long)_text;
|
||||
|
||||
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && phys_seed != 0) {
|
||||
/*
|
||||
* If KASLR is enabled, and we have some randomness available,
|
||||
* locate the kernel at a randomized offset in physical memory.
|
||||
*/
|
||||
status = efi_random_alloc(*reserve_size, min_kimg_align,
|
||||
reserve_addr, phys_seed,
|
||||
EFI_LOADER_CODE, EFI_ALLOC_LIMIT);
|
||||
if (status != EFI_SUCCESS)
|
||||
efi_warn("efi_random_alloc() failed: 0x%lx\n", status);
|
||||
} else {
|
||||
status = EFI_OUT_OF_RESOURCES;
|
||||
}
|
||||
|
||||
if (status != EFI_SUCCESS) {
|
||||
if (!check_image_region((u64)_text, kernel_memsize)) {
|
||||
efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n");
|
||||
} else if (IS_ALIGNED((u64)_text, min_kimg_align) &&
|
||||
(u64)_end < EFI_ALLOC_LIMIT) {
|
||||
/*
|
||||
* Just execute from wherever we were loaded by the
|
||||
* UEFI PE/COFF loader if the placement is suitable.
|
||||
*/
|
||||
*image_addr = (u64)_text;
|
||||
*reserve_size = 0;
|
||||
return EFI_SUCCESS;
|
||||
}
|
||||
|
||||
status = efi_allocate_pages_aligned(*reserve_size, reserve_addr,
|
||||
ULONG_MAX, min_kimg_align,
|
||||
EFI_LOADER_CODE);
|
||||
|
||||
if (status != EFI_SUCCESS) {
|
||||
efi_err("Failed to relocate kernel\n");
|
||||
*reserve_size = 0;
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
*image_addr = *reserve_addr;
|
||||
memcpy((void *)*image_addr, _text, kernel_size);
|
||||
caches_clean_inval_pou(*image_addr, *image_addr + kernel_codesize);
|
||||
efi_remap_image(*image_addr, *reserve_size, kernel_codesize);
|
||||
status = efi_kaslr_relocate_kernel(image_addr,
|
||||
reserve_addr, reserve_size,
|
||||
kernel_size, kernel_codesize,
|
||||
kernel_memsize,
|
||||
efi_kaslr_get_phys_seed(image_handle));
|
||||
if (status != EFI_SUCCESS)
|
||||
return status;
|
||||
|
||||
return EFI_SUCCESS;
|
||||
}
|
||||
|
|
@ -159,3 +63,8 @@ unsigned long primary_entry_offset(void)
|
|||
*/
|
||||
return (char *)primary_entry - _text;
|
||||
}
|
||||
|
||||
void efi_icache_sync(unsigned long start, unsigned long end)
|
||||
{
|
||||
caches_clean_inval_pou(start, end);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1133,6 +1133,14 @@ const u8 *__efi_get_smbios_string(const struct efi_smbios_record *record,
|
|||
|
||||
void efi_remap_image(unsigned long image_base, unsigned alloc_size,
|
||||
unsigned long code_size);
|
||||
efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr,
|
||||
unsigned long *reserve_addr,
|
||||
unsigned long *reserve_size,
|
||||
unsigned long kernel_size,
|
||||
unsigned long kernel_codesize,
|
||||
unsigned long kernel_memsize,
|
||||
u32 phys_seed);
|
||||
u32 efi_kaslr_get_phys_seed(efi_handle_t image_handle);
|
||||
|
||||
asmlinkage efi_status_t __efiapi
|
||||
efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Helper functions used by the EFI stub on multiple
|
||||
* architectures to deal with physical address space randomization.
|
||||
*/
|
||||
#include <linux/efi.h>
|
||||
|
||||
#include "efistub.h"
|
||||
|
||||
/**
|
||||
* efi_kaslr_get_phys_seed() - Get random seed for physical kernel KASLR
|
||||
* @image_handle: Handle to the image
|
||||
*
|
||||
* If KASLR is not disabled, obtain a random seed using EFI_RNG_PROTOCOL
|
||||
* that will be used to move the kernel physical mapping.
|
||||
*
|
||||
* Return: the random seed
|
||||
*/
|
||||
u32 efi_kaslr_get_phys_seed(efi_handle_t image_handle)
|
||||
{
|
||||
efi_status_t status;
|
||||
u32 phys_seed;
|
||||
efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID;
|
||||
void *p;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
|
||||
return 0;
|
||||
|
||||
if (efi_nokaslr) {
|
||||
efi_info("KASLR disabled on kernel command line\n");
|
||||
} else if (efi_bs_call(handle_protocol, image_handle,
|
||||
&li_fixed_proto, &p) == EFI_SUCCESS) {
|
||||
efi_info("Image placement fixed by loader\n");
|
||||
} else {
|
||||
status = efi_get_random_bytes(sizeof(phys_seed),
|
||||
(u8 *)&phys_seed);
|
||||
if (status == EFI_SUCCESS) {
|
||||
return phys_seed;
|
||||
} else if (status == EFI_NOT_FOUND) {
|
||||
efi_info("EFI_RNG_PROTOCOL unavailable\n");
|
||||
efi_nokaslr = true;
|
||||
} else if (status != EFI_SUCCESS) {
|
||||
efi_err("efi_get_random_bytes() failed (0x%lx)\n",
|
||||
status);
|
||||
efi_nokaslr = true;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Distro versions of GRUB may ignore the BSS allocation entirely (i.e., fail
|
||||
* to provide space, and fail to zero it). Check for this condition by double
|
||||
* checking that the first and the last byte of the image are covered by the
|
||||
* same EFI memory map entry.
|
||||
*/
|
||||
static bool check_image_region(u64 base, u64 size)
|
||||
{
|
||||
struct efi_boot_memmap *map;
|
||||
efi_status_t status;
|
||||
bool ret = false;
|
||||
int map_offset;
|
||||
|
||||
status = efi_get_memory_map(&map, false);
|
||||
if (status != EFI_SUCCESS)
|
||||
return false;
|
||||
|
||||
for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) {
|
||||
efi_memory_desc_t *md = (void *)map->map + map_offset;
|
||||
u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE;
|
||||
|
||||
/*
|
||||
* Find the region that covers base, and return whether
|
||||
* it covers base+size bytes.
|
||||
*/
|
||||
if (base >= md->phys_addr && base < end) {
|
||||
ret = (base + size) <= end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
efi_bs_call(free_pool, map);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* efi_kaslr_relocate_kernel() - Relocate the kernel (random if KASLR enabled)
|
||||
* @image_addr: Pointer to the current kernel location
|
||||
* @reserve_addr: Pointer to the relocated kernel location
|
||||
* @reserve_size: Size of the relocated kernel
|
||||
* @kernel_size: Size of the text + data
|
||||
* @kernel_codesize: Size of the text
|
||||
* @kernel_memsize: Size of the text + data + bss
|
||||
* @phys_seed: Random seed used for the relocation
|
||||
*
|
||||
* If KASLR is not enabled, this function relocates the kernel to a fixed
|
||||
* address (or leave it as its current location). If KASLR is enabled, the
|
||||
* kernel physical location is randomized using the seed in parameter.
|
||||
*
|
||||
* Return: status code, EFI_SUCCESS if relocation is successful
|
||||
*/
|
||||
efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr,
|
||||
unsigned long *reserve_addr,
|
||||
unsigned long *reserve_size,
|
||||
unsigned long kernel_size,
|
||||
unsigned long kernel_codesize,
|
||||
unsigned long kernel_memsize,
|
||||
u32 phys_seed)
|
||||
{
|
||||
efi_status_t status;
|
||||
u64 min_kimg_align = efi_get_kimg_min_align();
|
||||
|
||||
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && phys_seed != 0) {
|
||||
/*
|
||||
* If KASLR is enabled, and we have some randomness available,
|
||||
* locate the kernel at a randomized offset in physical memory.
|
||||
*/
|
||||
status = efi_random_alloc(*reserve_size, min_kimg_align,
|
||||
reserve_addr, phys_seed,
|
||||
EFI_LOADER_CODE, EFI_ALLOC_LIMIT);
|
||||
if (status != EFI_SUCCESS)
|
||||
efi_warn("efi_random_alloc() failed: 0x%lx\n", status);
|
||||
} else {
|
||||
status = EFI_OUT_OF_RESOURCES;
|
||||
}
|
||||
|
||||
if (status != EFI_SUCCESS) {
|
||||
if (!check_image_region(*image_addr, kernel_memsize)) {
|
||||
efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n");
|
||||
} else if (IS_ALIGNED(*image_addr, min_kimg_align) &&
|
||||
(unsigned long)_end < EFI_ALLOC_LIMIT) {
|
||||
/*
|
||||
* Just execute from wherever we were loaded by the
|
||||
* UEFI PE/COFF loader if the placement is suitable.
|
||||
*/
|
||||
*reserve_size = 0;
|
||||
return EFI_SUCCESS;
|
||||
}
|
||||
|
||||
status = efi_allocate_pages_aligned(*reserve_size, reserve_addr,
|
||||
ULONG_MAX, min_kimg_align,
|
||||
EFI_LOADER_CODE);
|
||||
|
||||
if (status != EFI_SUCCESS) {
|
||||
efi_err("Failed to relocate kernel\n");
|
||||
*reserve_size = 0;
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy((void *)*reserve_addr, (void *)*image_addr, kernel_size);
|
||||
*image_addr = *reserve_addr;
|
||||
efi_icache_sync(*image_addr, *image_addr + kernel_codesize);
|
||||
efi_remap_image(*image_addr, *reserve_size, kernel_codesize);
|
||||
|
||||
return status;
|
||||
}
|
||||
|
|
@ -30,32 +30,29 @@ efi_status_t handle_kernel_image(unsigned long *image_addr,
|
|||
efi_loaded_image_t *image,
|
||||
efi_handle_t image_handle)
|
||||
{
|
||||
unsigned long kernel_size = 0;
|
||||
unsigned long preferred_addr;
|
||||
unsigned long kernel_size, kernel_codesize, kernel_memsize;
|
||||
efi_status_t status;
|
||||
|
||||
kernel_size = _edata - _start;
|
||||
kernel_codesize = __init_text_end - _start;
|
||||
kernel_memsize = kernel_size + (_end - _edata);
|
||||
*image_addr = (unsigned long)_start;
|
||||
*image_size = kernel_size + (_end - _edata);
|
||||
|
||||
/*
|
||||
* RISC-V kernel maps PAGE_OFFSET virtual address to the same physical
|
||||
* address where kernel is booted. That's why kernel should boot from
|
||||
* as low as possible to avoid wastage of memory. Currently, dram_base
|
||||
* is occupied by the firmware. So the preferred address for kernel to
|
||||
* boot is next aligned address. If preferred address is not available,
|
||||
* relocate_kernel will fall back to efi_low_alloc_above to allocate
|
||||
* lowest possible memory region as long as the address and size meets
|
||||
* the alignment constraints.
|
||||
*/
|
||||
preferred_addr = EFI_KIMG_PREFERRED_ADDRESS;
|
||||
status = efi_relocate_kernel(image_addr, kernel_size, *image_size,
|
||||
preferred_addr, efi_get_kimg_min_align(),
|
||||
0x0);
|
||||
*image_size = kernel_memsize;
|
||||
*reserve_size = *image_size;
|
||||
|
||||
status = efi_kaslr_relocate_kernel(image_addr,
|
||||
reserve_addr, reserve_size,
|
||||
kernel_size, kernel_codesize, kernel_memsize,
|
||||
efi_kaslr_get_phys_seed(image_handle));
|
||||
if (status != EFI_SUCCESS) {
|
||||
efi_err("Failed to relocate kernel\n");
|
||||
*image_size = 0;
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
void efi_icache_sync(unsigned long start, unsigned long end)
|
||||
{
|
||||
asm volatile ("fence.i" ::: "memory");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -334,6 +334,11 @@ if RISCV
|
|||
config ARCH_R9A07G043
|
||||
bool "RISC-V Platform support for RZ/Five"
|
||||
select ARCH_RZG2L
|
||||
select AX45MP_L2_CACHE if RISCV_DMA_NONCOHERENT
|
||||
select DMA_GLOBAL_POOL
|
||||
select ERRATA_ANDES if RISCV_SBI
|
||||
select ERRATA_ANDES_CMO if ERRATA_ANDES
|
||||
|
||||
help
|
||||
This enables support for the Renesas RZ/Five SoC.
|
||||
|
||||
|
|
|
|||
|
|
@ -445,6 +445,8 @@ typedef struct elf64_shdr {
|
|||
#define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */
|
||||
#define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode */
|
||||
#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */
|
||||
#define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */
|
||||
#define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */
|
||||
#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */
|
||||
#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */
|
||||
#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */
|
||||
|
|
|
|||
|
|
@ -870,7 +870,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
|
|||
GFP_KERNEL);
|
||||
if (!pack)
|
||||
return NULL;
|
||||
pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);
|
||||
pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
|
||||
if (!pack->ptr) {
|
||||
kfree(pack);
|
||||
return NULL;
|
||||
|
|
@ -894,7 +894,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
|
|||
mutex_lock(&pack_mutex);
|
||||
if (size > BPF_PROG_PACK_SIZE) {
|
||||
size = round_up(size, PAGE_SIZE);
|
||||
ptr = module_alloc(size);
|
||||
ptr = bpf_jit_alloc_exec(size);
|
||||
if (ptr) {
|
||||
bpf_fill_ill_insns(ptr, size);
|
||||
set_vm_flush_reset_perms(ptr);
|
||||
|
|
@ -932,7 +932,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr)
|
|||
|
||||
mutex_lock(&pack_mutex);
|
||||
if (hdr->size > BPF_PROG_PACK_SIZE) {
|
||||
module_memfree(hdr);
|
||||
bpf_jit_free_exec(hdr);
|
||||
goto out;
|
||||
}
|
||||
|
||||
|
|
@ -956,7 +956,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr)
|
|||
if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
|
||||
BPF_PROG_CHUNK_COUNT, 0) == 0) {
|
||||
list_del(&pack->list);
|
||||
module_memfree(pack->ptr);
|
||||
bpf_jit_free_exec(pack->ptr);
|
||||
kfree(pack);
|
||||
}
|
||||
out:
|
||||
|
|
|
|||
Loading…
Reference in New Issue