diff --git a/arch/x86/CMakeLists.txt b/arch/x86/CMakeLists.txt index fc14df5714c..757c007172b 100644 --- a/arch/x86/CMakeLists.txt +++ b/arch/x86/CMakeLists.txt @@ -44,11 +44,12 @@ set(GENIDT ${ZEPHYR_BASE}/scripts/gen_idt.py) define_property(GLOBAL PROPERTY PROPERTY_OUTPUT_ARCH BRIEF_DOCS " " FULL_DOCS " ") -# Use gen_idt.py and objcopy to generate irq_int_vector_map.o and -# staticIdt.o from the elf file zephyr_prebuilt +# Use gen_idt.py and objcopy to generate irq_int_vector_map.o, +# irq_vectors_alloc.o, and staticIdt.o from the elf file zephyr_prebuilt set(gen_idt_output_files ${CMAKE_CURRENT_BINARY_DIR}/irq_int_vector_map.bin ${CMAKE_CURRENT_BINARY_DIR}/staticIdt.bin + ${CMAKE_CURRENT_BINARY_DIR}/irq_vectors_alloc.bin ) add_custom_target( gen_idt_output @@ -56,13 +57,14 @@ add_custom_target( ${gen_idt_output_files} ) add_custom_command( - OUTPUT irq_int_vector_map.bin staticIdt.bin + OUTPUT irq_int_vector_map.bin staticIdt.bin irq_vectors_alloc.bin COMMAND ${PYTHON_EXECUTABLE} ${GENIDT} --kernel $ --output-idt staticIdt.bin --vector-map irq_int_vector_map.bin + --output-vectors-alloc irq_vectors_alloc.bin ${GENIDT_EXTRA_ARGS} DEPENDS zephyr_prebuilt WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} @@ -100,22 +102,39 @@ add_custom_command( DEPENDS gen_idt_output staticIdt.bin WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) - +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/irq_vectors_alloc.o + COMMAND + ${CMAKE_OBJCOPY} + -I binary + -B ${OUTPUT_ARCH} + -O ${OUTPUT_FORMAT} + --rename-section .data=irq_vectors_alloc_data + irq_vectors_alloc.bin + irq_vectors_alloc.o + DEPENDS gen_idt_output irq_vectors_alloc.bin + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) add_custom_target(irq_int_vector_map_o DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/irq_int_vector_map.o) add_custom_target(staticIdt_o DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/staticIdt.o) +add_custom_target(irq_vectors_alloc_o DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/irq_vectors_alloc.o) add_library(irq_int_vector_map STATIC IMPORTED GLOBAL) add_library(staticIdt STATIC IMPORTED GLOBAL) +add_library(irq_vectors_alloc STATIC IMPORTED GLOBAL) set_property(TARGET irq_int_vector_map PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/irq_int_vector_map.o) set_property(TARGET staticIdt PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/staticIdt.o) +set_property(TARGET irq_vectors_alloc PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/irq_vectors_alloc.o) add_dependencies(irq_int_vector_map irq_int_vector_map_o) add_dependencies(staticIdt staticIdt_o) +add_dependencies(irq_vectors_alloc irq_vectors_alloc_o) set_property(GLOBAL APPEND PROPERTY GENERATED_KERNEL_OBJECT_FILES irq_int_vector_map) set_property(GLOBAL APPEND PROPERTY GENERATED_KERNEL_OBJECT_FILES staticIdt) +set_property(GLOBAL APPEND PROPERTY GENERATED_KERNEL_OBJECT_FILES irq_vectors_alloc) if(CONFIG_X86_MMU) # Use gen_mmu.py and objcopy to generate mmu_tables.o from from the diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 8e3224812b2..88d777dc839 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -285,6 +285,14 @@ config IRQ_OFFLOAD_VECTOR where there is a fixed IRQ-to-vector mapping another value may be needed to avoid collision. +config X86_DYNAMIC_IRQ_STUBS + int "Number of dynamic interrupt stubs" + depends on DYNAMIC_INTERRUPTS + default 4 + help + Installing interrupt handlers with irq_connect_dynamic() requires + some stub code to be generated at build time, one stub per dynamic + interrupt. config XIP default n diff --git a/arch/x86/core/crt0.S b/arch/x86/core/crt0.S index 2aa2f63f41e..4dbb57a32e8 100644 --- a/arch/x86/core/crt0.S +++ b/arch/x86/core/crt0.S @@ -24,7 +24,7 @@ GDATA(_idt_base_address) GDATA(_interrupt_stack) - GDATA(_Idt) + GDATA(z_x86_idt) #ifndef CONFIG_GDT_DYNAMIC GDATA(_gdt) #endif @@ -173,7 +173,6 @@ SECTION_FUNC(TEXT_START, __start) #if CONFIG_SET_GDT lgdt _gdt_rom /* load 32-bit operand size GDT */ #endif - lidt _Idt /* load 32-bit operand size IDT */ @@ -379,6 +378,8 @@ __csSet: mov $MAIN_TSS, %ax ltr %ax #endif + lidt z_x86_idt /* load 32-bit operand size IDT */ + /* Jump to C portion of kernel initialization and never return */ jmp _Cstart @@ -458,7 +459,7 @@ _sse_mxcsr_default_value: /* Interrupt Descriptor Table (IDT) definition */ -_Idt: +z_x86_idt: .word (CONFIG_IDT_NUM_VECTORS * 8) - 1 /* limit: size of IDT-1 */ /* diff --git a/arch/x86/core/intstub.S b/arch/x86/core/intstub.S index b0703caadef..6ea17fad009 100644 --- a/arch/x86/core/intstub.S +++ b/arch/x86/core/intstub.S @@ -26,6 +26,7 @@ GTEXT(_SpuriousIntNoErrCodeHandler) GTEXT(_SpuriousIntHandler) GTEXT(_irq_sw_handler) + GTEXT(z_dynamic_stubs_begin) /* externs */ @@ -464,3 +465,62 @@ SECTION_FUNC(TEXT, _irq_sw_handler) jmp _interrupt_enter #endif + +#if CONFIG_X86_DYNAMIC_IRQ_STUBS > 0 +z_dynamic_irq_stub_common: + /* stub number already pushed */ + push $z_x86_dynamic_irq_handler + jmp _interrupt_enter + +/* Create all the dynamic IRQ stubs + * + * NOTE: Please update DYN_STUB_SIZE in include/arch/x86/arch.h if you change + * how large the generated stubs are, otherwise _get_dynamic_stub() will + * be unable to correctly determine the offset + */ + +/* + * Create nice labels for all the stubs so we can see where we + * are in a debugger + */ +.altmacro +.macro __INT_STUB_NUM id +z_dynamic_irq_stub_\id: +.endm +.macro INT_STUB_NUM id +__INT_STUB_NUM %id +.endm + +z_dynamic_stubs_begin: +stub_num = 0 +.rept ((CONFIG_X86_DYNAMIC_IRQ_STUBS + Z_DYN_STUB_PER_BLOCK - 1) / Z_DYN_STUB_PER_BLOCK) + block_counter = 0 + .rept Z_DYN_STUB_PER_BLOCK + .if stub_num < CONFIG_X86_DYNAMIC_IRQ_STUBS + INT_STUB_NUM stub_num + /* + * 2-byte push imm8. + */ + push $stub_num + + /* + * Check to make sure this isn't the last stub in + * a block, in which case we just fall through + */ + .if (block_counter <> (Z_DYN_STUB_PER_BLOCK - 1) && \ + (stub_num <> CONFIG_X86_DYNAMIC_IRQ_STUBS - 1)) + /* This should always be a 2-byte jmp rel8 */ + jmp 1f + .endif + stub_num = stub_num + 1 + block_counter = block_counter + 1 + .endif + .endr + /* + * This must a 5-bvte jump rel32, which is why z_dynamic_irq_stub_common + * is before the actual stubs + */ +1: jmp z_dynamic_irq_stub_common +.endr +#endif /* CONFIG_X86_DYNAMIC_IRQ_STUBS > 0 */ + diff --git a/arch/x86/core/irq_manage.c b/arch/x86/core/irq_manage.c index 9281920c0b6..2f5b85c6525 100644 --- a/arch/x86/core/irq_manage.c +++ b/arch/x86/core/irq_manage.c @@ -22,6 +22,7 @@ #include #include #include +#include extern void _SpuriousIntHandler(void *); extern void _SpuriousIntNoErrCodeHandler(void *); @@ -37,7 +38,6 @@ void *__attribute__((section(".spurNoErrIsr"))) MK_ISR_NAME(_SpuriousIntNoErrCodeHandler) = &_SpuriousIntNoErrCodeHandler; - /* FIXME: IRQ direct inline functions have to be placed here and not in * arch/cpu.h as inline functions due to nasty circular dependency between * arch/cpu.h and kernel_structs.h; the inline functions typically need to @@ -99,3 +99,251 @@ void _arch_isr_direct_footer(int swap) } } +#if CONFIG_X86_DYNAMIC_IRQ_STUBS > 0 + +/* + * z_interrupt_vectors_allocated[] bitfield is generated by the 'gen_idt' tool. + * It is initialized to identify which interrupts have been statically + * connected and which interrupts are available to be dynamically connected at + * run time, with a 1 bit indicating a free vector. The variable itself is + * defined in the linker file. + */ +extern unsigned int z_interrupt_vectors_allocated[]; + +struct dyn_irq_info { + /** IRQ handler */ + void (*handler)(void *param); + /** Parameter to pass to the handler */ + void *param; +}; + +/* + * Instead of creating a large sparse table mapping all possible IDT vectors + * to dyn_irq_info, the dynamic stubs push a "stub id" onto the stack + * which is used by common_dynamic_handler() to fetch the appropriate + * information out of this much smaller table + */ +static struct dyn_irq_info dyn_irq_list[CONFIG_X86_DYNAMIC_IRQ_STUBS]; +static unsigned int next_irq_stub; + +/* Memory address pointing to where in ROM the code for the dynamic stubs are. + * Linker symbol. + */ +extern char z_dynamic_stubs_begin[]; + +#ifndef CONFIG_X86_FIXED_IRQ_MAPPING +/** + * @brief Allocate a free interrupt vector given + * + * This routine scans the z_interrupt_vectors_allocated[] array for a free vector + * that satisfies the specified . + * + * This routine assumes that the relationship between interrupt priority and + * interrupt vector is : + * + * priority = (vector / 16) - 2; + * + * Vectors 0 to 31 are reserved for CPU exceptions and do NOT fall under + * the priority scheme. The first vector used for priority level 0 will be 32. + * Each interrupt priority level contains 16 vectors. + * + * It is also assumed that the interrupt controllers are capable of managing + * interrupt requests on a per-vector level as opposed to a per-priority level. + * For example, the local APIC on Pentium4 and later processors, the in-service + * register (ISR) and the interrupt request register (IRR) are 256 bits wide. + * + * @return allocated interrupt vector + */ + +static unsigned int priority_to_free_vector(unsigned int requested_priority) +{ + unsigned int entry; + unsigned int fsb; /* first set bit in entry */ + unsigned int search_set; + unsigned int vector_block; + unsigned int vector; + + static unsigned int mask[2] = {0x0000ffff, 0xffff0000}; + + vector_block = requested_priority + 2; + + __ASSERT(((vector_block << 4) + 15) <= CONFIG_IDT_NUM_VECTORS, + "IDT too small (%d entries) to use priority %d", + CONFIG_IDT_NUM_VECTORS, requested_priority); + + /* + * Atomically allocate a vector from the + * z_interrupt_vectors_allocated[] array to prevent race conditions + * with other threads attempting to allocate an interrupt + * vector. + * + * Note: As z_interrupt_vectors_allocated[] is initialized by the + * 'gen_idt.py' tool, it is critical that this routine use the same + * algorithm as the 'gen_idt.py' tool for allocating interrupt vectors. + */ + + entry = vector_block >> 1; + + /* + * The z_interrupt_vectors_allocated[] entry indexed by 'entry' + * is a 32-bit quantity and thus represents the vectors for a pair of + * priority levels. Mask out the unwanted priority level and then use + * find_lsb_set() to scan for an available vector of the requested + * priority. + * + * Note that find_lsb_set() returns bit position from 1 to 32, or 0 if + * the argument is zero. + */ + search_set = mask[vector_block & 1] & + z_interrupt_vectors_allocated[entry]; + fsb = find_lsb_set(search_set); + + __ASSERT(fsb != 0, "No remaning vectors for priority level %d", + requested_priority); + + /* + * An available vector of the requested priority was found. + * Mark it as allocated by clearing the bit. + */ + --fsb; + z_interrupt_vectors_allocated[entry] &= ~(1 << fsb); + + /* compute vector given allocated bit within the priority level */ + vector = (entry << 5) + fsb; + + return vector; +} +#endif /* !CONFIG_X86_FIXED_IRQ_MAPPING */ + +/** + * @brief Get the memory address of an unused dynamic IRQ or exception stub + * + * We generate at build time a set of dynamic stubs which push + * a stub index onto the stack for use as an argument by + * common handling code. + * + * @param stub_idx Stub number to fetch the corresponding stub function + * @return Pointer to the stub code to install into the IDT + */ +static void *get_dynamic_stub(int stub_idx) +{ + u32_t offset; + + /* + * Because we want the sizes of the stubs to be consisent and minimized, + * stubs are grouped into blocks, each containing a push and subsequent + * 2-byte jump instruction to the end of the block, which then contains + * a larger jump instruction to common dynamic IRQ handling code + */ + offset = (stub_idx * Z_DYN_STUB_SIZE) + + ((stub_idx / Z_DYN_STUB_PER_BLOCK) * + Z_DYN_STUB_LONG_JMP_EXTRA_SIZE); + + return (void *)((u32_t)&z_dynamic_stubs_begin + offset); +} + +extern const struct pseudo_descriptor z_x86_idt; + +static void idt_vector_install(int vector, void *irq_handler) +{ + int key; + + key = irq_lock(); + _init_irq_gate(&z_x86_idt.entries[vector], CODE_SEG, + (u32_t)irq_handler, 0); +#ifdef CONFIG_MVIC + /* MVIC requires IDT be reloaded if the entries table is ever changed */ + _set_idt(&z_x86_idt); +#endif + irq_unlock(key); +} + +/** + * + * @brief Connect a C routine to a hardware interrupt + * + * @param irq virtualized IRQ to connect to + * @param priority requested priority of interrupt + * @param routine the C interrupt handler + * @param parameter parameter passed to C routine + * @param flags IRQ flags + * + * This routine connects an interrupt service routine (ISR) coded in C to + * the specified hardware . An interrupt vector will be allocated to + * satisfy the specified . + * + * The specified represents a virtualized IRQ, i.e. it does not + * necessarily represent a specific IRQ line on a given interrupt controller + * device. The platform presents a virtualized set of IRQs from 0 to N, where + * N is the total number of IRQs supported by all the interrupt controller + * devices on the board. See the platform's documentation for the mapping of + * virtualized IRQ to physical IRQ. + * + * When the device asserts an interrupt on the specified , a switch to + * the interrupt stack is performed (if not already executing on the interrupt + * stack), followed by saving the integer (i.e. non-floating point) thread of + * the currently executing thread or ISR. The ISR specified by + * will then be invoked with the single . When the ISR returns, a + * context switch may occur. + * + * On some platforms parameter needs to be specified to indicate if + * the irq is triggered by low or high level or by rising or falling edge. + * + * The routine searches for the first available element in the dynamic_stubs + * array and uses it for the stub. + * + * @return the allocated interrupt vector + * + * WARNINGS + * This routine does not perform range checking on the requested + * and thus, depending on the underlying interrupt controller, may result + * in the assignment of an interrupt vector located in the reserved range of + * the processor. + */ + +int _arch_irq_connect_dynamic(unsigned int irq, unsigned int priority, + void (*routine)(void *parameter), void *parameter, + u32_t flags) +{ + int vector, stub_idx, key; + + key = irq_lock(); + +#ifdef CONFIG_X86_FIXED_IRQ_MAPPING + vector = _IRQ_TO_INTERRUPT_VECTOR(irq); +#else + vector = priority_to_free_vector(priority); + /* 0 indicates not used, vectors for interrupts start at 32 */ + __ASSERT(_irq_to_interrupt_vector[irq] == 0, + "IRQ %d already configured", irq); + _irq_to_interrupt_vector[irq] = vector; +#endif + _irq_controller_irq_config(vector, irq, flags); + + stub_idx = next_irq_stub++; + __ASSERT(stub_idx < CONFIG_X86_DYNAMIC_IRQ_STUBS, + "No available interrupt stubs found"); + + dyn_irq_list[stub_idx].handler = routine; + dyn_irq_list[stub_idx].param = parameter; + idt_vector_install(vector, get_dynamic_stub(stub_idx)); + + irq_unlock(key); + + return vector; +} + +/** + * @brief Common dynamic IRQ handler function + * + * This gets called by the IRQ entry asm code with the stub index supplied as + * an argument. Look up the required information in dyn_irq_list and + * execute it. + * + * @param stub_idx Index into the dyn_irq_list array + */ +void z_x86_dynamic_irq_handler(u8_t stub_idx) +{ + dyn_irq_list[stub_idx].handler(dyn_irq_list[stub_idx].param); +} +#endif /* CONFIG_X86_DYNAMIC_IRQ_STUBS > 0 */ diff --git a/arch/x86/include/kernel_arch_func.h b/arch/x86/include/kernel_arch_func.h index 421f52ab55c..252cd31b123 100644 --- a/arch/x86/include/kernel_arch_func.h +++ b/arch/x86/include/kernel_arch_func.h @@ -128,9 +128,6 @@ static inline void _IntLibInit(void) { } -/* the _idt_base_address symbol is generated via a linker script */ -extern unsigned char _idt_base_address[]; - extern FUNC_NORETURN void _x86_userspace_enter(k_thread_entry_t user_entry, void *p1, void *p2, void *p3, u32_t stack_end, diff --git a/include/arch/x86/arch.h b/include/arch/x86/arch.h index b36d9539f67..dc0ec94a567 100644 --- a/include/arch/x86/arch.h +++ b/include/arch/x86/arch.h @@ -43,6 +43,12 @@ extern "C" { */ #define MK_ISR_NAME(x) __isr__##x +#define Z_DYN_STUB_SIZE 4 +#define Z_DYN_STUB_OFFSET 0 +#define Z_DYN_STUB_LONG_JMP_EXTRA_SIZE 3 +#define Z_DYN_STUB_PER_BLOCK 32 + + #ifndef _ASMLANGUAGE #ifdef CONFIG_INT_LATENCY_BENCHMARK @@ -55,6 +61,7 @@ void _int_latency_stop(void); /* interrupt/exception/error related definitions */ + /* * The TCS must be aligned to the same boundary as that used by the floating * point register set. This applies even for threads that don't initially diff --git a/include/arch/x86/linker.ld b/include/arch/x86/linker.ld index 0cd66057486..62b34964487 100644 --- a/include/arch/x86/linker.ld +++ b/include/arch/x86/linker.ld @@ -117,13 +117,15 @@ SECTIONS *(.rodata) *(".rodata.*") *(.gnu.linkonce.r.*) + +#ifndef CONFIG_DYNAMIC_INTERRUPTS . = ALIGN(8); _idt_base_address = .; #ifdef LINKER_PASS2 KEEP(*(staticIdt)) #else . += CONFIG_IDT_NUM_VECTORS * 8; -#endif +#endif /* LINKER_PASS2 */ #ifndef CONFIG_X86_FIXED_IRQ_MAPPING . = ALIGN(4); @@ -133,7 +135,8 @@ SECTIONS #else . += CONFIG_MAX_IRQ_LINES; #endif -#endif +#endif /* CONFIG_X86_FIXED_IRQ_MAPPING */ +#endif /* CONFIG_DYNAMIC_INTERRUPTS */ #ifdef CONFIG_SOC_RODATA_LD #include @@ -268,6 +271,34 @@ SECTIONS KERNEL_INPUT_SECTION(".data.*") *(".kernel.*") +#ifdef CONFIG_DYNAMIC_INTERRUPTS + . = ALIGN(8); + _idt_base_address = .; +#ifdef LINKER_PASS2 + KEEP(*(staticIdt)) +#else + . += CONFIG_IDT_NUM_VECTORS * 8; +#endif /* LINKER_PASS2 */ + +#ifndef CONFIG_X86_FIXED_IRQ_MAPPING + . = ALIGN(4); + _irq_to_interrupt_vector = .; +#ifdef LINKER_PASS2 + KEEP(*(irq_int_vector_map)) +#else + . += CONFIG_MAX_IRQ_LINES; +#endif /* LINKER_PASS2 */ +#endif /* CONFIG_X86_FIXED_IRQ_MAPPING */ + + z_interrupt_vectors_allocated = .; +#ifdef LINKER_PASS2 + KEEP(*(irq_vectors_alloc_data)) +#else + . += (CONFIG_IDT_NUM_VECTORS + 7) / 8; +#endif /* LINKER_PASS2 */ +#endif /* CONFIG_DYNAMIC_INTERRUPTS */ + + #ifdef CONFIG_SOC_RWDATA_LD #include #endif diff --git a/include/arch/x86/segmentation.h b/include/arch/x86/segmentation.h index 9276372dbbf..0ab50c438cd 100644 --- a/include/arch/x86/segmentation.h +++ b/include/arch/x86/segmentation.h @@ -377,6 +377,8 @@ struct __packed far_ptr { extern struct pseudo_descriptor _gdt; #endif +extern const struct pseudo_descriptor z_idt; + /** * Properly set the segment descriptor segment and offset * diff --git a/scripts/gen_idt.py b/scripts/gen_idt.py index c7c3eb01487..b70bbd58e04 100755 --- a/scripts/gen_idt.py +++ b/scripts/gen_idt.py @@ -235,6 +235,8 @@ def parse_args(): help="Output file mapping IRQ lines to IDT vectors") parser.add_argument("-o", "--output-idt", required=True, help="Output file containing IDT binary") + parser.add_argument("-a", "--output-vectors-alloc", required=False, + help="Output file indicating allocated vectors") parser.add_argument("-k", "--kernel", required=True, help="Zephyr kernel image") parser.add_argument("-v", "--verbose", action="store_true", @@ -244,6 +246,27 @@ def parse_args(): args.verbose = 1 +def create_irq_vectors_allocated(vectors, spur_code, spur_nocode, filename): + # Construct a bitfield over all the IDT vectors, where if bit n is 1, + # that vector is free. those vectors have either of the two spurious + # interrupt handlers installed, they are free for runtime installation + # of interrupts + num_chars = (len(vectors) + 7) // 8 + vbits = [0 for i in range(num_chars)] + for i in range(len(vectors)): + handler, _, _ = vectors[i] + if handler != spur_code and handler != spur_nocode: + continue + + vbit_index = i // 8 + vbit_val = 1 << (i % 8) + vbits[vbit_index] = vbits[vbit_index] | vbit_val + + with open(filename, "wb") as fp: + for char in vbits: + fp.write(struct.pack("