/* * Copyright © 2014 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /* * This lowering pass converts references to input/output variables with * loads/stores to actual input/output intrinsics. */ #include "nir.h" #include "nir_builder.h" #include "nir_deref.h" #include "nir_xfb_info.h" #include "util/u_math.h" struct lower_io_state { void *dead_ctx; nir_builder builder; int (*type_size)(const struct glsl_type *type, bool); nir_variable_mode modes; nir_lower_io_options options; struct set variable_names; }; static const char * add_variable_name(struct lower_io_state *state, const char *name) { if (!name) return NULL; bool found = false; struct set_entry *entry = _mesa_set_search_or_add(&state->variable_names, name, &found); if (!found) entry->key = (void*)ralloc_strdup(state->builder.shader, name); return entry->key; } static nir_intrinsic_op ssbo_atomic_for_deref(nir_intrinsic_op deref_op) { switch (deref_op) { case nir_intrinsic_deref_atomic: return nir_intrinsic_ssbo_atomic; case nir_intrinsic_deref_atomic_swap: return nir_intrinsic_ssbo_atomic_swap; default: unreachable("Invalid SSBO atomic"); } } static nir_intrinsic_op global_atomic_for_deref(nir_address_format addr_format, nir_intrinsic_op deref_op) { switch (deref_op) { case nir_intrinsic_deref_atomic: if (addr_format != nir_address_format_2x32bit_global) return nir_intrinsic_global_atomic; else return nir_intrinsic_global_atomic_2x32; case nir_intrinsic_deref_atomic_swap: if (addr_format != nir_address_format_2x32bit_global) return nir_intrinsic_global_atomic_swap; else return nir_intrinsic_global_atomic_swap_2x32; default: unreachable("Invalid SSBO atomic"); } } static nir_intrinsic_op shared_atomic_for_deref(nir_intrinsic_op deref_op) { switch (deref_op) { case nir_intrinsic_deref_atomic: return nir_intrinsic_shared_atomic; case nir_intrinsic_deref_atomic_swap: return nir_intrinsic_shared_atomic_swap; default: unreachable("Invalid shared atomic"); } } static nir_intrinsic_op task_payload_atomic_for_deref(nir_intrinsic_op deref_op) { switch (deref_op) { case nir_intrinsic_deref_atomic: return nir_intrinsic_task_payload_atomic; case nir_intrinsic_deref_atomic_swap: return nir_intrinsic_task_payload_atomic_swap; default: unreachable("Invalid task payload atomic"); } } void nir_assign_var_locations(nir_shader *shader, nir_variable_mode mode, unsigned *size, int (*type_size)(const struct glsl_type *, bool)) { unsigned location = 0; nir_foreach_variable_with_modes(var, shader, mode) { var->data.driver_location = location; bool bindless_type_size = var->data.mode == nir_var_shader_in || var->data.mode == nir_var_shader_out || var->data.bindless; location += type_size(var->type, bindless_type_size); } *size = location; } /** * Some inputs and outputs are arrayed, meaning that there is an extra level * of array indexing to handle mismatches between the shader interface and the * dispatch pattern of the shader. For instance, geometry shaders are * executed per-primitive while their inputs and outputs are specified * per-vertex so all inputs and outputs have to be additionally indexed with * the vertex index within the primitive. */ bool nir_is_arrayed_io(const nir_variable *var, gl_shader_stage stage) { if (var->data.patch || !glsl_type_is_array(var->type)) return false; if (stage == MESA_SHADER_MESH) { /* NV_mesh_shader: this is flat array for the whole workgroup. */ if (var->data.location == VARYING_SLOT_PRIMITIVE_INDICES) return var->data.per_primitive; } if (var->data.mode == nir_var_shader_in) { if (var->data.per_vertex) { assert(stage == MESA_SHADER_FRAGMENT); return true; } return stage == MESA_SHADER_GEOMETRY || stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL; } if (var->data.mode == nir_var_shader_out) return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_MESH; return false; } static bool uses_high_dvec2_semantic(struct lower_io_state *state, const nir_variable *var) { return state->builder.shader->info.stage == MESA_SHADER_VERTEX && state->options & nir_lower_io_lower_64bit_to_32_new && var->data.mode == nir_var_shader_in && glsl_type_is_dual_slot(glsl_without_array(var->type)); } static unsigned get_number_of_slots(struct lower_io_state *state, const nir_variable *var) { const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, state->builder.shader->info.stage)) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } /* NV_mesh_shader: * PRIMITIVE_INDICES is a flat array, not a proper arrayed output, * as opposed to D3D-style mesh shaders where it's addressed by * the primitive index. * Prevent assigning several slots to primitive indices, * to avoid some issues. */ if (state->builder.shader->info.stage == MESA_SHADER_MESH && var->data.location == VARYING_SLOT_PRIMITIVE_INDICES && !nir_is_arrayed_io(var, state->builder.shader->info.stage)) return 1; return state->type_size(type, var->data.bindless) / (uses_high_dvec2_semantic(state, var) ? 2 : 1); } static nir_def * get_io_offset(nir_builder *b, nir_deref_instr *deref, nir_def **array_index, int (*type_size)(const struct glsl_type *, bool), unsigned *component, bool bts) { nir_deref_path path; nir_deref_path_init(&path, deref, NULL); assert(path.path[0]->deref_type == nir_deref_type_var); nir_deref_instr **p = &path.path[1]; /* For arrayed I/O (e.g., per-vertex input arrays in geometry shader * inputs), skip the outermost array index. Process the rest normally. */ if (array_index != NULL) { assert((*p)->deref_type == nir_deref_type_array); *array_index = (*p)->arr.index.ssa; p++; } if (path.path[0]->var->data.compact && nir_src_is_const((*p)->arr.index)) { assert((*p)->deref_type == nir_deref_type_array); assert(glsl_type_is_scalar((*p)->type)); /* We always lower indirect dereferences for "compact" array vars. */ const unsigned index = nir_src_as_uint((*p)->arr.index); const unsigned total_offset = *component + index; const unsigned slot_offset = total_offset / 4; *component = total_offset % 4; return nir_imm_int(b, type_size(glsl_vec4_type(), bts) * slot_offset); } /* Just emit code and let constant-folding go to town */ nir_def *offset = nir_imm_int(b, 0); for (; *p; p++) { if ((*p)->deref_type == nir_deref_type_array) { unsigned size = type_size((*p)->type, bts); nir_def *mul = nir_amul_imm(b, (*p)->arr.index.ssa, size); offset = nir_iadd(b, offset, mul); } else if ((*p)->deref_type == nir_deref_type_struct) { /* p starts at path[1], so this is safe */ nir_deref_instr *parent = *(p - 1); unsigned field_offset = 0; for (unsigned i = 0; i < (*p)->strct.index; i++) { field_offset += type_size(glsl_get_struct_field(parent->type, i), bts); } offset = nir_iadd_imm(b, offset, field_offset); } else { unreachable("Unsupported deref type"); } } nir_deref_path_finish(&path); return offset; } static bool is_medium_precision(const nir_shader *shader, const nir_variable *var) { if (shader->options->io_options & nir_io_mediump_is_32bit) return false; return var->data.precision == GLSL_PRECISION_MEDIUM || var->data.precision == GLSL_PRECISION_LOW; } static nir_def * emit_load(struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, unsigned num_components, unsigned bit_size, nir_alu_type dest_type, bool high_dvec2) { nir_builder *b = &state->builder; const nir_shader *nir = b->shader; nir_variable_mode mode = var->data.mode; nir_def *barycentric = NULL; nir_intrinsic_op op; switch (mode) { case nir_var_shader_in: if (nir->info.stage == MESA_SHADER_FRAGMENT && nir->options->use_interpolated_input_intrinsics && var->data.interpolation != INTERP_MODE_FLAT && !var->data.per_primitive) { if (var->data.interpolation == INTERP_MODE_EXPLICIT || var->data.per_vertex) { assert(array_index != NULL); op = nir_intrinsic_load_input_vertex; } else { assert(array_index == NULL); nir_intrinsic_op bary_op; if (var->data.sample) bary_op = nir_intrinsic_load_barycentric_sample; else if (var->data.centroid) bary_op = nir_intrinsic_load_barycentric_centroid; else bary_op = nir_intrinsic_load_barycentric_pixel; barycentric = nir_load_barycentric(&state->builder, bary_op, var->data.interpolation); op = nir_intrinsic_load_interpolated_input; } } else { if (var->data.per_primitive) op = nir_intrinsic_load_per_primitive_input; else if (array_index) op = nir_intrinsic_load_per_vertex_input; else op = nir_intrinsic_load_input; } break; case nir_var_shader_out: op = !array_index ? nir_intrinsic_load_output : var->data.per_primitive ? nir_intrinsic_load_per_primitive_output : nir_intrinsic_load_per_vertex_output; break; case nir_var_uniform: op = nir_intrinsic_load_uniform; break; default: unreachable("Unknown variable mode"); } nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->builder.shader, op); load->num_components = num_components; load->name = add_variable_name(state, var->name); nir_intrinsic_set_base(load, var->data.driver_location); if (nir_intrinsic_has_range(load)) { const struct glsl_type *type = var->type; if (array_index) type = glsl_get_array_element(type); unsigned var_size = state->type_size(type, var->data.bindless); nir_intrinsic_set_range(load, var_size); } if (mode == nir_var_shader_in || mode == nir_var_shader_out) nir_intrinsic_set_component(load, component); if (nir_intrinsic_has_access(load)) nir_intrinsic_set_access(load, var->data.access); nir_intrinsic_set_dest_type(load, dest_type); if (load->intrinsic != nir_intrinsic_load_uniform) { nir_io_semantics semantics = { 0 }; semantics.location = var->data.location; semantics.num_slots = get_number_of_slots(state, var); semantics.fb_fetch_output = var->data.fb_fetch_output; semantics.medium_precision = is_medium_precision(b->shader, var); semantics.high_dvec2 = high_dvec2; /* "per_vertex" is misnamed. It means "explicit interpolation with * the original vertex order", which is a stricter version of * INTERP_MODE_EXPLICIT. */ semantics.interp_explicit_strict = var->data.per_vertex; nir_intrinsic_set_io_semantics(load, semantics); } if (array_index) { load->src[0] = nir_src_for_ssa(array_index); load->src[1] = nir_src_for_ssa(offset); } else if (barycentric) { load->src[0] = nir_src_for_ssa(barycentric); load->src[1] = nir_src_for_ssa(offset); } else { load->src[0] = nir_src_for_ssa(offset); } nir_def_init(&load->instr, &load->def, num_components, bit_size); nir_builder_instr_insert(b, &load->instr); return &load->def; } static nir_def * lower_load(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { const bool lower_double = !glsl_type_is_integer(type) && state->options & nir_lower_io_lower_64bit_float_to_32; if (intrin->def.bit_size == 64 && (lower_double || (state->options & (nir_lower_io_lower_64bit_to_32_new | nir_lower_io_lower_64bit_to_32)))) { nir_builder *b = &state->builder; bool use_high_dvec2_semantic = uses_high_dvec2_semantic(state, var); /* Each slot is a dual slot, so divide the offset within the variable * by 2. */ if (use_high_dvec2_semantic) offset = nir_ushr_imm(b, offset, 1); const unsigned slot_size = state->type_size(glsl_dvec_type(2), false); nir_def *comp64[4]; assert(component == 0 || component == 2); unsigned dest_comp = 0; bool high_dvec2 = false; while (dest_comp < intrin->def.num_components) { const unsigned num_comps = MIN2(intrin->def.num_components - dest_comp, (4 - component) / 2); nir_def *data32 = emit_load(state, array_index, var, offset, component, num_comps * 2, 32, nir_type_uint32, high_dvec2); for (unsigned i = 0; i < num_comps; i++) { comp64[dest_comp + i] = nir_pack_64_2x32(b, nir_channels(b, data32, 3 << (i * 2))); } /* Only the first store has a component offset */ component = 0; dest_comp += num_comps; if (use_high_dvec2_semantic) { /* Increment the offset when we wrap around the dual slot. */ if (high_dvec2) offset = nir_iadd_imm(b, offset, slot_size); high_dvec2 = !high_dvec2; } else { offset = nir_iadd_imm(b, offset, slot_size); } } return nir_vec(b, comp64, intrin->def.num_components); } else if (intrin->def.bit_size == 1) { /* Booleans are 32-bit */ assert(glsl_type_is_boolean(type)); return nir_b2b1(&state->builder, emit_load(state, array_index, var, offset, component, intrin->def.num_components, 32, nir_type_bool32, false)); } else { return emit_load(state, array_index, var, offset, component, intrin->def.num_components, intrin->def.bit_size, nir_get_nir_type_for_glsl_type(type), false); } } static void emit_store(struct lower_io_state *state, nir_def *data, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, unsigned num_components, nir_component_mask_t write_mask, nir_alu_type src_type) { nir_builder *b = &state->builder; assert(var->data.mode == nir_var_shader_out); nir_intrinsic_op op = !array_index ? nir_intrinsic_store_output : var->data.per_primitive ? nir_intrinsic_store_per_primitive_output : nir_intrinsic_store_per_vertex_output; nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->builder.shader, op); store->num_components = num_components; store->name = add_variable_name(state, var->name); store->src[0] = nir_src_for_ssa(data); const struct glsl_type *type = var->type; if (array_index) type = glsl_get_array_element(type); unsigned var_size = state->type_size(type, var->data.bindless); nir_intrinsic_set_base(store, var->data.driver_location); nir_intrinsic_set_range(store, var_size); nir_intrinsic_set_component(store, component); nir_intrinsic_set_src_type(store, src_type); nir_intrinsic_set_write_mask(store, write_mask); if (nir_intrinsic_has_access(store)) nir_intrinsic_set_access(store, var->data.access); if (array_index) store->src[1] = nir_src_for_ssa(array_index); store->src[array_index ? 2 : 1] = nir_src_for_ssa(offset); unsigned gs_streams = 0; if (state->builder.shader->info.stage == MESA_SHADER_GEOMETRY) { if (var->data.stream & NIR_STREAM_PACKED) { gs_streams = var->data.stream & ~NIR_STREAM_PACKED; } else { assert(var->data.stream < 4); gs_streams = 0; for (unsigned i = 0; i < num_components; ++i) gs_streams |= var->data.stream << (2 * i); } } nir_io_semantics semantics = { 0 }; semantics.location = var->data.location; semantics.num_slots = get_number_of_slots(state, var); semantics.dual_source_blend_index = var->data.index; semantics.gs_streams = gs_streams; semantics.medium_precision = is_medium_precision(b->shader, var); semantics.per_view = var->data.per_view; semantics.invariant = var->data.invariant; nir_intrinsic_set_io_semantics(store, semantics); nir_builder_instr_insert(b, &store->instr); } static void lower_store(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { const bool lower_double = !glsl_type_is_integer(type) && state->options & nir_lower_io_lower_64bit_float_to_32; if (intrin->src[1].ssa->bit_size == 64 && (lower_double || (state->options & (nir_lower_io_lower_64bit_to_32 | nir_lower_io_lower_64bit_to_32_new)))) { nir_builder *b = &state->builder; const unsigned slot_size = state->type_size(glsl_dvec_type(2), false); assert(component == 0 || component == 2); unsigned src_comp = 0; nir_component_mask_t write_mask = nir_intrinsic_write_mask(intrin); while (src_comp < intrin->num_components) { const unsigned num_comps = MIN2(intrin->num_components - src_comp, (4 - component) / 2); if (write_mask & BITFIELD_MASK(num_comps)) { nir_def *data = nir_channels(b, intrin->src[1].ssa, BITFIELD_RANGE(src_comp, num_comps)); nir_def *data32 = nir_bitcast_vector(b, data, 32); uint32_t write_mask32 = 0; for (unsigned i = 0; i < num_comps; i++) { if (write_mask & BITFIELD_MASK(num_comps) & (1 << i)) write_mask32 |= 3 << (i * 2); } emit_store(state, data32, array_index, var, offset, component, data32->num_components, write_mask32, nir_type_uint32); } /* Only the first store has a component offset */ component = 0; src_comp += num_comps; write_mask >>= num_comps; offset = nir_iadd_imm(b, offset, slot_size); } } else if (intrin->def.bit_size == 1) { /* Booleans are 32-bit */ assert(glsl_type_is_boolean(type)); nir_def *b32_val = nir_b2b32(&state->builder, intrin->src[1].ssa); emit_store(state, b32_val, array_index, var, offset, component, intrin->num_components, nir_intrinsic_write_mask(intrin), nir_type_bool32); } else { emit_store(state, intrin->src[1].ssa, array_index, var, offset, component, intrin->num_components, nir_intrinsic_write_mask(intrin), nir_get_nir_type_for_glsl_type(type)); } } static nir_def * lower_interpolate_at(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { nir_builder *b = &state->builder; assert(var->data.mode == nir_var_shader_in); /* Ignore interpolateAt() for flat variables - flat is flat. Lower * interpolateAtVertex() for explicit variables. */ if (var->data.interpolation == INTERP_MODE_FLAT || var->data.interpolation == INTERP_MODE_EXPLICIT) { nir_def *vertex_index = NULL; if (var->data.interpolation == INTERP_MODE_EXPLICIT) { assert(intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex); vertex_index = intrin->src[1].ssa; } return lower_load(intrin, state, vertex_index, var, offset, component, type); } /* None of the supported APIs allow interpolation on 64-bit things */ assert(intrin->def.bit_size <= 32); nir_intrinsic_op bary_op; switch (intrin->intrinsic) { case nir_intrinsic_interp_deref_at_centroid: bary_op = nir_intrinsic_load_barycentric_centroid; break; case nir_intrinsic_interp_deref_at_sample: bary_op = nir_intrinsic_load_barycentric_at_sample; break; case nir_intrinsic_interp_deref_at_offset: bary_op = nir_intrinsic_load_barycentric_at_offset; break; default: unreachable("Bogus interpolateAt() intrinsic."); } nir_intrinsic_instr *bary_setup = nir_intrinsic_instr_create(state->builder.shader, bary_op); nir_def_init(&bary_setup->instr, &bary_setup->def, 2, 32); nir_intrinsic_set_interp_mode(bary_setup, var->data.interpolation); if (intrin->intrinsic == nir_intrinsic_interp_deref_at_sample || intrin->intrinsic == nir_intrinsic_interp_deref_at_offset || intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex) bary_setup->src[0] = nir_src_for_ssa(intrin->src[1].ssa); nir_builder_instr_insert(b, &bary_setup->instr); nir_io_semantics semantics = { 0 }; semantics.location = var->data.location; semantics.num_slots = get_number_of_slots(state, var); semantics.medium_precision = is_medium_precision(b->shader, var); nir_def *load = nir_load_interpolated_input(&state->builder, intrin->def.num_components, intrin->def.bit_size, &bary_setup->def, offset, .base = var->data.driver_location, .component = component, .io_semantics = semantics, .dest_type = nir_type_float | intrin->def.bit_size); return load; } static bool nir_lower_io_block(nir_block *block, struct lower_io_state *state) { nir_builder *b = &state->builder; const nir_shader_compiler_options *options = b->shader->options; bool progress = false; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: case nir_intrinsic_store_deref: /* We can lower the io for this nir instrinsic */ break; case nir_intrinsic_interp_deref_at_centroid: case nir_intrinsic_interp_deref_at_sample: case nir_intrinsic_interp_deref_at_offset: case nir_intrinsic_interp_deref_at_vertex: /* We can optionally lower these to load_interpolated_input */ if (options->use_interpolated_input_intrinsics || options->lower_interpolate_at) break; FALLTHROUGH; default: /* We can't lower the io for this nir instrinsic, so skip it */ continue; } nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (!nir_deref_mode_is_one_of(deref, state->modes)) continue; nir_variable *var = nir_deref_instr_get_variable(deref); b->cursor = nir_before_instr(instr); const bool is_arrayed = nir_is_arrayed_io(var, b->shader->info.stage); nir_def *offset; nir_def *array_index = NULL; unsigned component_offset = var->data.location_frac; bool bindless_type_size = var->data.mode == nir_var_shader_in || var->data.mode == nir_var_shader_out || var->data.bindless; if (nir_deref_instr_is_known_out_of_bounds(deref)) { /* Section 5.11 (Out-of-Bounds Accesses) of the GLSL 4.60 spec says: * * In the subsections described above for array, vector, matrix and * structure accesses, any out-of-bounds access produced undefined * behavior.... * Out-of-bounds reads return undefined values, which * include values from other variables of the active program or zero. * Out-of-bounds writes may be discarded or overwrite * other variables of the active program. * * GL_KHR_robustness and GL_ARB_robustness encourage us to return zero * for reads. * * Otherwise get_io_offset would return out-of-bound offset which may * result in out-of-bound loading/storing of inputs/outputs, * that could cause issues in drivers down the line. */ if (intrin->intrinsic != nir_intrinsic_store_deref) { nir_def *zero = nir_imm_zero(b, intrin->def.num_components, intrin->def.bit_size); nir_def_rewrite_uses(&intrin->def, zero); } nir_instr_remove(&intrin->instr); progress = true; continue; } offset = get_io_offset(b, deref, is_arrayed ? &array_index : NULL, state->type_size, &component_offset, bindless_type_size); nir_def *replacement = NULL; switch (intrin->intrinsic) { case nir_intrinsic_load_deref: replacement = lower_load(intrin, state, array_index, var, offset, component_offset, deref->type); break; case nir_intrinsic_store_deref: lower_store(intrin, state, array_index, var, offset, component_offset, deref->type); break; case nir_intrinsic_interp_deref_at_centroid: case nir_intrinsic_interp_deref_at_sample: case nir_intrinsic_interp_deref_at_offset: case nir_intrinsic_interp_deref_at_vertex: assert(array_index == NULL); replacement = lower_interpolate_at(intrin, state, var, offset, component_offset, deref->type); break; default: continue; } if (replacement) { nir_def_rewrite_uses(&intrin->def, replacement); } nir_instr_remove(&intrin->instr); progress = true; } return progress; } static bool nir_lower_io_impl(nir_function_impl *impl, nir_variable_mode modes, int (*type_size)(const struct glsl_type *, bool), nir_lower_io_options options) { struct lower_io_state state; bool progress = false; state.builder = nir_builder_create(impl); state.dead_ctx = ralloc_context(NULL); state.modes = modes; state.type_size = type_size; state.options = options; _mesa_set_init(&state.variable_names, state.dead_ctx, _mesa_hash_string, _mesa_key_string_equal); ASSERTED nir_variable_mode supported_modes = nir_var_shader_in | nir_var_shader_out | nir_var_uniform; assert(!(modes & ~supported_modes)); nir_foreach_block(block, impl) { progress |= nir_lower_io_block(block, &state); } ralloc_free(state.dead_ctx); nir_metadata_preserve(impl, nir_metadata_none); return progress; } /** Lower load/store_deref intrinsics on I/O variables to offset-based intrinsics * * This pass is intended to be used for cross-stage shader I/O and driver- * managed uniforms to turn deref-based access into a simpler model using * locations or offsets. For fragment shader inputs, it can optionally turn * load_deref into an explicit interpolation using barycentrics coming from * one of the load_barycentric_* intrinsics. This pass requires that all * deref chains are complete and contain no casts. */ bool nir_lower_io(nir_shader *shader, nir_variable_mode modes, int (*type_size)(const struct glsl_type *, bool), nir_lower_io_options options) { bool progress = false; nir_foreach_function_impl(impl, shader) { progress |= nir_lower_io_impl(impl, modes, type_size, options); } return progress; } static unsigned type_scalar_size_bytes(const struct glsl_type *type) { assert(glsl_type_is_vector_or_scalar(type) || glsl_type_is_matrix(type)); return glsl_type_is_boolean(type) ? 4 : glsl_get_bit_size(type) / 8; } nir_def * nir_build_addr_iadd(nir_builder *b, nir_def *addr, nir_address_format addr_format, nir_variable_mode modes, nir_def *offset) { assert(offset->num_components == 1); switch (addr_format) { case nir_address_format_32bit_global: case nir_address_format_64bit_global: case nir_address_format_32bit_offset: assert(addr->bit_size == offset->bit_size); assert(addr->num_components == 1); return nir_iadd(b, addr, offset); case nir_address_format_2x32bit_global: { assert(addr->num_components == 2); nir_def *lo = nir_channel(b, addr, 0); nir_def *hi = nir_channel(b, addr, 1); nir_def *res_lo = nir_iadd(b, lo, offset); nir_def *carry = nir_b2i32(b, nir_ult(b, res_lo, lo)); nir_def *res_hi = nir_iadd(b, hi, carry); return nir_vec2(b, res_lo, res_hi); } case nir_address_format_32bit_offset_as_64bit: assert(addr->num_components == 1); assert(offset->bit_size == 32); return nir_u2u64(b, nir_iadd(b, nir_u2u32(b, addr), offset)); case nir_address_format_64bit_global_32bit_offset: case nir_address_format_64bit_bounded_global: assert(addr->num_components == 4); assert(addr->bit_size == offset->bit_size); return nir_vector_insert_imm(b, addr, nir_iadd(b, nir_channel(b, addr, 3), offset), 3); case nir_address_format_32bit_index_offset: assert(addr->num_components == 2); assert(addr->bit_size == offset->bit_size); return nir_vector_insert_imm(b, addr, nir_iadd(b, nir_channel(b, addr, 1), offset), 1); case nir_address_format_32bit_index_offset_pack64: assert(addr->num_components == 1); assert(offset->bit_size == 32); return nir_pack_64_2x32_split(b, nir_iadd(b, nir_unpack_64_2x32_split_x(b, addr), offset), nir_unpack_64_2x32_split_y(b, addr)); case nir_address_format_vec2_index_32bit_offset: assert(addr->num_components == 3); assert(offset->bit_size == 32); return nir_vector_insert_imm(b, addr, nir_iadd(b, nir_channel(b, addr, 2), offset), 2); case nir_address_format_62bit_generic: assert(addr->num_components == 1); assert(addr->bit_size == 64); assert(offset->bit_size == 64); if (!(modes & ~(nir_var_function_temp | nir_var_shader_temp | nir_var_mem_shared))) { /* If we're sure it's one of these modes, we can do an easy 32-bit * addition and don't need to bother with 64-bit math. */ nir_def *addr32 = nir_unpack_64_2x32_split_x(b, addr); nir_def *type = nir_unpack_64_2x32_split_y(b, addr); addr32 = nir_iadd(b, addr32, nir_u2u32(b, offset)); return nir_pack_64_2x32_split(b, addr32, type); } else { return nir_iadd(b, addr, offset); } case nir_address_format_logical: unreachable("Unsupported address format"); } unreachable("Invalid address format"); } static unsigned addr_get_offset_bit_size(nir_def *addr, nir_address_format addr_format) { if (addr_format == nir_address_format_32bit_offset_as_64bit || addr_format == nir_address_format_32bit_index_offset_pack64) return 32; return addr->bit_size; } nir_def * nir_build_addr_iadd_imm(nir_builder *b, nir_def *addr, nir_address_format addr_format, nir_variable_mode modes, int64_t offset) { if (!offset) return addr; return nir_build_addr_iadd( b, addr, addr_format, modes, nir_imm_intN_t(b, offset, addr_get_offset_bit_size(addr, addr_format))); } static nir_def * build_addr_for_var(nir_builder *b, nir_variable *var, nir_address_format addr_format) { assert(var->data.mode & (nir_var_uniform | nir_var_mem_shared | nir_var_mem_task_payload | nir_var_mem_global | nir_var_shader_temp | nir_var_function_temp | nir_var_mem_push_const | nir_var_mem_constant)); const unsigned num_comps = nir_address_format_num_components(addr_format); const unsigned bit_size = nir_address_format_bit_size(addr_format); switch (addr_format) { case nir_address_format_2x32bit_global: case nir_address_format_32bit_global: case nir_address_format_64bit_global: { nir_def *base_addr; switch (var->data.mode) { case nir_var_shader_temp: base_addr = nir_load_scratch_base_ptr(b, num_comps, bit_size, 0); break; case nir_var_function_temp: base_addr = nir_load_scratch_base_ptr(b, num_comps, bit_size, 1); break; case nir_var_mem_constant: base_addr = nir_load_constant_base_ptr(b, num_comps, bit_size); break; case nir_var_mem_shared: base_addr = nir_load_shared_base_ptr(b, num_comps, bit_size); break; case nir_var_mem_global: base_addr = nir_load_global_base_ptr(b, num_comps, bit_size); break; default: unreachable("Unsupported variable mode"); } return nir_build_addr_iadd_imm(b, base_addr, addr_format, var->data.mode, var->data.driver_location); } case nir_address_format_32bit_offset: assert(var->data.driver_location <= UINT32_MAX); return nir_imm_int(b, var->data.driver_location); case nir_address_format_32bit_offset_as_64bit: assert(var->data.driver_location <= UINT32_MAX); return nir_imm_int64(b, var->data.driver_location); case nir_address_format_62bit_generic: switch (var->data.mode) { case nir_var_shader_temp: case nir_var_function_temp: assert(var->data.driver_location <= UINT32_MAX); return nir_imm_intN_t(b, var->data.driver_location | 2ull << 62, 64); case nir_var_mem_shared: assert(var->data.driver_location <= UINT32_MAX); return nir_imm_intN_t(b, var->data.driver_location | 1ull << 62, 64); case nir_var_mem_global: return nir_iadd_imm(b, nir_load_global_base_ptr(b, num_comps, bit_size), var->data.driver_location); default: unreachable("Unsupported variable mode"); } default: unreachable("Unsupported address format"); } } static nir_def * build_runtime_addr_mode_check(nir_builder *b, nir_def *addr, nir_address_format addr_format, nir_variable_mode mode) { /* The compile-time check failed; do a run-time check */ switch (addr_format) { case nir_address_format_62bit_generic: { assert(addr->num_components == 1); assert(addr->bit_size == 64); nir_def *mode_enum = nir_ushr_imm(b, addr, 62); switch (mode) { case nir_var_function_temp: case nir_var_shader_temp: return nir_ieq_imm(b, mode_enum, 0x2); case nir_var_mem_shared: return nir_ieq_imm(b, mode_enum, 0x1); case nir_var_mem_global: return nir_ior(b, nir_ieq_imm(b, mode_enum, 0x0), nir_ieq_imm(b, mode_enum, 0x3)); default: unreachable("Invalid mode check intrinsic"); } } default: unreachable("Unsupported address mode"); } } unsigned nir_address_format_bit_size(nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_global: return 32; case nir_address_format_2x32bit_global: return 32; case nir_address_format_64bit_global: return 64; case nir_address_format_64bit_global_32bit_offset: return 32; case nir_address_format_64bit_bounded_global: return 32; case nir_address_format_32bit_index_offset: return 32; case nir_address_format_32bit_index_offset_pack64: return 64; case nir_address_format_vec2_index_32bit_offset: return 32; case nir_address_format_62bit_generic: return 64; case nir_address_format_32bit_offset: return 32; case nir_address_format_32bit_offset_as_64bit: return 64; case nir_address_format_logical: return 32; } unreachable("Invalid address format"); } unsigned nir_address_format_num_components(nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_global: return 1; case nir_address_format_2x32bit_global: return 2; case nir_address_format_64bit_global: return 1; case nir_address_format_64bit_global_32bit_offset: return 4; case nir_address_format_64bit_bounded_global: return 4; case nir_address_format_32bit_index_offset: return 2; case nir_address_format_32bit_index_offset_pack64: return 1; case nir_address_format_vec2_index_32bit_offset: return 3; case nir_address_format_62bit_generic: return 1; case nir_address_format_32bit_offset: return 1; case nir_address_format_32bit_offset_as_64bit: return 1; case nir_address_format_logical: return 1; } unreachable("Invalid address format"); } static nir_def * addr_to_index(nir_builder *b, nir_def *addr, nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_index_offset: assert(addr->num_components == 2); return nir_channel(b, addr, 0); case nir_address_format_32bit_index_offset_pack64: return nir_unpack_64_2x32_split_y(b, addr); case nir_address_format_vec2_index_32bit_offset: assert(addr->num_components == 3); return nir_trim_vector(b, addr, 2); default: unreachable("Invalid address format"); } } static nir_def * addr_to_offset(nir_builder *b, nir_def *addr, nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_index_offset: assert(addr->num_components == 2); return nir_channel(b, addr, 1); case nir_address_format_32bit_index_offset_pack64: return nir_unpack_64_2x32_split_x(b, addr); case nir_address_format_vec2_index_32bit_offset: assert(addr->num_components == 3); return nir_channel(b, addr, 2); case nir_address_format_32bit_offset: return addr; case nir_address_format_32bit_offset_as_64bit: case nir_address_format_62bit_generic: return nir_u2u32(b, addr); default: unreachable("Invalid address format"); } } /** Returns true if the given address format resolves to a global address */ static bool addr_format_is_global(nir_address_format addr_format, nir_variable_mode mode) { if (addr_format == nir_address_format_62bit_generic) return mode == nir_var_mem_global; return addr_format == nir_address_format_32bit_global || addr_format == nir_address_format_2x32bit_global || addr_format == nir_address_format_64bit_global || addr_format == nir_address_format_64bit_global_32bit_offset || addr_format == nir_address_format_64bit_bounded_global; } static bool addr_format_is_offset(nir_address_format addr_format, nir_variable_mode mode) { if (addr_format == nir_address_format_62bit_generic) return mode != nir_var_mem_global; return addr_format == nir_address_format_32bit_offset || addr_format == nir_address_format_32bit_offset_as_64bit; } static nir_def * addr_to_global(nir_builder *b, nir_def *addr, nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_global: case nir_address_format_64bit_global: case nir_address_format_62bit_generic: assert(addr->num_components == 1); return addr; case nir_address_format_2x32bit_global: assert(addr->num_components == 2); return addr; case nir_address_format_64bit_global_32bit_offset: case nir_address_format_64bit_bounded_global: assert(addr->num_components == 4); return nir_iadd(b, nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2)), nir_u2u64(b, nir_channel(b, addr, 3))); case nir_address_format_32bit_index_offset: case nir_address_format_32bit_index_offset_pack64: case nir_address_format_vec2_index_32bit_offset: case nir_address_format_32bit_offset: case nir_address_format_32bit_offset_as_64bit: case nir_address_format_logical: unreachable("Cannot get a 64-bit address with this address format"); } unreachable("Invalid address format"); } static bool addr_format_needs_bounds_check(nir_address_format addr_format) { return addr_format == nir_address_format_64bit_bounded_global; } static nir_def * addr_is_in_bounds(nir_builder *b, nir_def *addr, nir_address_format addr_format, unsigned size) { assert(addr_format == nir_address_format_64bit_bounded_global); assert(addr->num_components == 4); assert(size > 0); return nir_ult(b, nir_iadd_imm(b, nir_channel(b, addr, 3), size - 1), nir_channel(b, addr, 2)); } static void nir_get_explicit_deref_range(nir_deref_instr *deref, nir_address_format addr_format, uint32_t *out_base, uint32_t *out_range) { uint32_t base = 0; uint32_t range = glsl_get_explicit_size(deref->type, false); while (true) { nir_deref_instr *parent = nir_deref_instr_parent(deref); switch (deref->deref_type) { case nir_deref_type_array: case nir_deref_type_array_wildcard: case nir_deref_type_ptr_as_array: { const unsigned stride = nir_deref_instr_array_stride(deref); if (stride == 0) goto fail; if (!parent) goto fail; if (deref->deref_type != nir_deref_type_array_wildcard && nir_src_is_const(deref->arr.index)) { base += stride * nir_src_as_uint(deref->arr.index); } else { if (glsl_get_length(parent->type) == 0) goto fail; range += stride * (glsl_get_length(parent->type) - 1); } break; } case nir_deref_type_struct: { if (!parent) goto fail; base += glsl_get_struct_field_offset(parent->type, deref->strct.index); break; } case nir_deref_type_cast: { nir_instr *parent_instr = deref->parent.ssa->parent_instr; switch (parent_instr->type) { case nir_instr_type_load_const: { nir_load_const_instr *load = nir_instr_as_load_const(parent_instr); switch (addr_format) { case nir_address_format_32bit_offset: base += load->value[1].u32; break; case nir_address_format_32bit_index_offset: base += load->value[1].u32; break; case nir_address_format_vec2_index_32bit_offset: base += load->value[2].u32; break; default: goto fail; } *out_base = base; *out_range = range; return; } case nir_instr_type_intrinsic: { nir_intrinsic_instr *intr = nir_instr_as_intrinsic(parent_instr); switch (intr->intrinsic) { case nir_intrinsic_load_vulkan_descriptor: /* Assume that a load_vulkan_descriptor won't contribute to an * offset within the resource. */ break; default: goto fail; } *out_base = base; *out_range = range; return; } default: goto fail; } } default: goto fail; } deref = parent; } fail: *out_base = 0; *out_range = ~0; } static nir_variable_mode canonicalize_generic_modes(nir_variable_mode modes) { assert(modes != 0); if (util_bitcount(modes) == 1) return modes; assert(!(modes & ~(nir_var_function_temp | nir_var_shader_temp | nir_var_mem_shared | nir_var_mem_global))); /* Canonicalize by converting shader_temp to function_temp */ if (modes & nir_var_shader_temp) { modes &= ~nir_var_shader_temp; modes |= nir_var_function_temp; } return modes; } static nir_intrinsic_op get_store_global_op_from_addr_format(nir_address_format addr_format) { if (addr_format != nir_address_format_2x32bit_global) return nir_intrinsic_store_global; else return nir_intrinsic_store_global_2x32; } static nir_intrinsic_op get_load_global_op_from_addr_format(nir_address_format addr_format) { if (addr_format != nir_address_format_2x32bit_global) return nir_intrinsic_load_global; else return nir_intrinsic_load_global_2x32; } static nir_intrinsic_op get_load_global_constant_op_from_addr_format(nir_address_format addr_format) { if (addr_format != nir_address_format_2x32bit_global) return nir_intrinsic_load_global_constant; else return nir_intrinsic_load_global_2x32; /* no dedicated op, fallback */ } static nir_def * build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *addr, nir_address_format addr_format, nir_variable_mode modes, uint32_t align_mul, uint32_t align_offset, unsigned num_components) { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); modes = canonicalize_generic_modes(modes); if (util_bitcount(modes) > 1) { if (addr_format_is_global(addr_format, modes)) { return build_explicit_io_load(b, intrin, addr, addr_format, nir_var_mem_global, align_mul, align_offset, num_components); } else if (modes & nir_var_function_temp) { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_function_temp)); nir_def *res1 = build_explicit_io_load(b, intrin, addr, addr_format, nir_var_function_temp, align_mul, align_offset, num_components); nir_push_else(b, NULL); nir_def *res2 = build_explicit_io_load(b, intrin, addr, addr_format, modes & ~nir_var_function_temp, align_mul, align_offset, num_components); nir_pop_if(b, NULL); return nir_if_phi(b, res1, res2); } else { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_mem_shared)); assert(modes & nir_var_mem_shared); nir_def *res1 = build_explicit_io_load(b, intrin, addr, addr_format, nir_var_mem_shared, align_mul, align_offset, num_components); nir_push_else(b, NULL); assert(modes & nir_var_mem_global); nir_def *res2 = build_explicit_io_load(b, intrin, addr, addr_format, nir_var_mem_global, align_mul, align_offset, num_components); nir_pop_if(b, NULL); return nir_if_phi(b, res1, res2); } } assert(util_bitcount(modes) == 1); const nir_variable_mode mode = modes; nir_intrinsic_op op; switch (intrin->intrinsic) { case nir_intrinsic_load_deref: switch (mode) { case nir_var_mem_ubo: if (addr_format == nir_address_format_64bit_global_32bit_offset) op = nir_intrinsic_load_global_constant_offset; else if (addr_format == nir_address_format_64bit_bounded_global) op = nir_intrinsic_load_global_constant_bounded; else if (addr_format_is_global(addr_format, mode)) op = nir_intrinsic_load_global_constant; else op = nir_intrinsic_load_ubo; break; case nir_var_mem_ssbo: if (addr_format_is_global(addr_format, mode)) op = nir_intrinsic_load_global; else op = nir_intrinsic_load_ssbo; break; case nir_var_mem_global: assert(addr_format_is_global(addr_format, mode)); op = get_load_global_op_from_addr_format(addr_format); break; case nir_var_uniform: assert(addr_format_is_offset(addr_format, mode)); assert(b->shader->info.stage == MESA_SHADER_KERNEL); op = nir_intrinsic_load_kernel_input; break; case nir_var_mem_shared: assert(addr_format_is_offset(addr_format, mode)); op = nir_intrinsic_load_shared; break; case nir_var_mem_task_payload: assert(addr_format_is_offset(addr_format, mode)); op = nir_intrinsic_load_task_payload; break; case nir_var_shader_temp: case nir_var_function_temp: if (addr_format_is_offset(addr_format, mode)) { op = nir_intrinsic_load_scratch; } else { assert(addr_format_is_global(addr_format, mode)); op = get_load_global_op_from_addr_format(addr_format); } break; case nir_var_mem_push_const: assert(addr_format == nir_address_format_32bit_offset); op = nir_intrinsic_load_push_constant; break; case nir_var_mem_constant: if (addr_format_is_offset(addr_format, mode)) { op = nir_intrinsic_load_constant; } else { assert(addr_format_is_global(addr_format, mode)); op = get_load_global_constant_op_from_addr_format(addr_format); } break; default: unreachable("Unsupported explicit IO variable mode"); } break; case nir_intrinsic_load_deref_block_intel: switch (mode) { case nir_var_mem_ssbo: if (addr_format_is_global(addr_format, mode)) op = nir_intrinsic_load_global_block_intel; else op = nir_intrinsic_load_ssbo_block_intel; break; case nir_var_mem_global: op = nir_intrinsic_load_global_block_intel; break; case nir_var_mem_shared: op = nir_intrinsic_load_shared_block_intel; break; default: unreachable("Unsupported explicit IO variable mode"); } break; default: unreachable("Invalid intrinsic"); } nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op); if (op == nir_intrinsic_load_global_constant_offset) { assert(addr_format == nir_address_format_64bit_global_32bit_offset); load->src[0] = nir_src_for_ssa( nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2))); load->src[1] = nir_src_for_ssa(nir_channel(b, addr, 3)); } else if (op == nir_intrinsic_load_global_constant_bounded) { assert(addr_format == nir_address_format_64bit_bounded_global); load->src[0] = nir_src_for_ssa( nir_pack_64_2x32(b, nir_trim_vector(b, addr, 2))); load->src[1] = nir_src_for_ssa(nir_channel(b, addr, 3)); load->src[2] = nir_src_for_ssa(nir_channel(b, addr, 2)); } else if (addr_format_is_global(addr_format, mode)) { load->src[0] = nir_src_for_ssa(addr_to_global(b, addr, addr_format)); } else if (addr_format_is_offset(addr_format, mode)) { assert(addr->num_components == 1); load->src[0] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } else { load->src[0] = nir_src_for_ssa(addr_to_index(b, addr, addr_format)); load->src[1] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } if (nir_intrinsic_has_access(load)) nir_intrinsic_set_access(load, nir_intrinsic_access(intrin)); if (op == nir_intrinsic_load_constant) { nir_intrinsic_set_base(load, 0); nir_intrinsic_set_range(load, b->shader->constant_data_size); } else if (op == nir_intrinsic_load_kernel_input) { nir_intrinsic_set_base(load, 0); nir_intrinsic_set_range(load, b->shader->num_uniforms); } else if (mode == nir_var_mem_push_const) { /* Push constants are required to be able to be chased back to the * variable so we can provide a base/range. */ nir_variable *var = nir_deref_instr_get_variable(deref); nir_intrinsic_set_base(load, 0); nir_intrinsic_set_range(load, glsl_get_explicit_size(var->type, false)); } unsigned bit_size = intrin->def.bit_size; if (bit_size == 1) { /* TODO: Make the native bool bit_size an option. */ bit_size = 32; } if (nir_intrinsic_has_align(load)) nir_intrinsic_set_align(load, align_mul, align_offset); if (nir_intrinsic_has_range_base(load)) { unsigned base, range; nir_get_explicit_deref_range(deref, addr_format, &base, &range); nir_intrinsic_set_range_base(load, base); nir_intrinsic_set_range(load, range); } load->num_components = num_components; nir_def_init(&load->instr, &load->def, num_components, bit_size); assert(bit_size % 8 == 0); nir_def *result; if (addr_format_needs_bounds_check(addr_format) && op != nir_intrinsic_load_global_constant_bounded) { /* We don't need to bounds-check global_constant_bounded because bounds * checking is handled by the intrinsic itself. * * The Vulkan spec for robustBufferAccess gives us quite a few options * as to what we can do with an OOB read. Unfortunately, returning * undefined values isn't one of them so we return an actual zero. */ nir_def *zero = nir_imm_zero(b, load->num_components, bit_size); /* TODO: Better handle block_intel. */ assert(load->num_components == 1); const unsigned load_size = bit_size / 8; nir_push_if(b, addr_is_in_bounds(b, addr, addr_format, load_size)); nir_builder_instr_insert(b, &load->instr); nir_pop_if(b, NULL); result = nir_if_phi(b, &load->def, zero); } else { nir_builder_instr_insert(b, &load->instr); result = &load->def; } if (intrin->def.bit_size == 1) { /* For shared, we can go ahead and use NIR's and/or the back-end's * standard encoding for booleans rather than forcing a 0/1 boolean. * This should save an instruction or two. */ if (mode == nir_var_mem_shared || mode == nir_var_shader_temp || mode == nir_var_function_temp) result = nir_b2b1(b, result); else result = nir_i2b(b, result); } return result; } static void build_explicit_io_store(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *addr, nir_address_format addr_format, nir_variable_mode modes, uint32_t align_mul, uint32_t align_offset, nir_def *value, nir_component_mask_t write_mask) { modes = canonicalize_generic_modes(modes); if (util_bitcount(modes) > 1) { if (addr_format_is_global(addr_format, modes)) { build_explicit_io_store(b, intrin, addr, addr_format, nir_var_mem_global, align_mul, align_offset, value, write_mask); } else if (modes & nir_var_function_temp) { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_function_temp)); build_explicit_io_store(b, intrin, addr, addr_format, nir_var_function_temp, align_mul, align_offset, value, write_mask); nir_push_else(b, NULL); build_explicit_io_store(b, intrin, addr, addr_format, modes & ~nir_var_function_temp, align_mul, align_offset, value, write_mask); nir_pop_if(b, NULL); } else { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_mem_shared)); assert(modes & nir_var_mem_shared); build_explicit_io_store(b, intrin, addr, addr_format, nir_var_mem_shared, align_mul, align_offset, value, write_mask); nir_push_else(b, NULL); assert(modes & nir_var_mem_global); build_explicit_io_store(b, intrin, addr, addr_format, nir_var_mem_global, align_mul, align_offset, value, write_mask); nir_pop_if(b, NULL); } return; } assert(util_bitcount(modes) == 1); const nir_variable_mode mode = modes; nir_intrinsic_op op; switch (intrin->intrinsic) { case nir_intrinsic_store_deref: assert(write_mask != 0); switch (mode) { case nir_var_mem_ssbo: if (addr_format_is_global(addr_format, mode)) op = get_store_global_op_from_addr_format(addr_format); else op = nir_intrinsic_store_ssbo; break; case nir_var_mem_global: assert(addr_format_is_global(addr_format, mode)); op = get_store_global_op_from_addr_format(addr_format); break; case nir_var_mem_shared: assert(addr_format_is_offset(addr_format, mode)); op = nir_intrinsic_store_shared; break; case nir_var_mem_task_payload: assert(addr_format_is_offset(addr_format, mode)); op = nir_intrinsic_store_task_payload; break; case nir_var_shader_temp: case nir_var_function_temp: if (addr_format_is_offset(addr_format, mode)) { op = nir_intrinsic_store_scratch; } else { assert(addr_format_is_global(addr_format, mode)); op = get_store_global_op_from_addr_format(addr_format); } break; default: unreachable("Unsupported explicit IO variable mode"); } break; case nir_intrinsic_store_deref_block_intel: assert(write_mask == 0); switch (mode) { case nir_var_mem_ssbo: if (addr_format_is_global(addr_format, mode)) op = nir_intrinsic_store_global_block_intel; else op = nir_intrinsic_store_ssbo_block_intel; break; case nir_var_mem_global: op = nir_intrinsic_store_global_block_intel; break; case nir_var_mem_shared: op = nir_intrinsic_store_shared_block_intel; break; default: unreachable("Unsupported explicit IO variable mode"); } break; default: unreachable("Invalid intrinsic"); } nir_intrinsic_instr *store = nir_intrinsic_instr_create(b->shader, op); if (value->bit_size == 1) { /* For shared, we can go ahead and use NIR's and/or the back-end's * standard encoding for booleans rather than forcing a 0/1 boolean. * This should save an instruction or two. * * TODO: Make the native bool bit_size an option. */ if (mode == nir_var_mem_shared || mode == nir_var_shader_temp || mode == nir_var_function_temp) value = nir_b2b32(b, value); else value = nir_b2iN(b, value, 32); } store->src[0] = nir_src_for_ssa(value); if (addr_format_is_global(addr_format, mode)) { store->src[1] = nir_src_for_ssa(addr_to_global(b, addr, addr_format)); } else if (addr_format_is_offset(addr_format, mode)) { assert(addr->num_components == 1); store->src[1] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } else { store->src[1] = nir_src_for_ssa(addr_to_index(b, addr, addr_format)); store->src[2] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } nir_intrinsic_set_write_mask(store, write_mask); if (nir_intrinsic_has_access(store)) nir_intrinsic_set_access(store, nir_intrinsic_access(intrin)); nir_intrinsic_set_align(store, align_mul, align_offset); assert(value->num_components == 1 || value->num_components == intrin->num_components); store->num_components = value->num_components; assert(value->bit_size % 8 == 0); if (addr_format_needs_bounds_check(addr_format)) { /* TODO: Better handle block_intel. */ assert(store->num_components == 1); const unsigned store_size = value->bit_size / 8; nir_push_if(b, addr_is_in_bounds(b, addr, addr_format, store_size)); nir_builder_instr_insert(b, &store->instr); nir_pop_if(b, NULL); } else { nir_builder_instr_insert(b, &store->instr); } } static nir_def * build_explicit_io_atomic(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *addr, nir_address_format addr_format, nir_variable_mode modes) { modes = canonicalize_generic_modes(modes); if (util_bitcount(modes) > 1) { if (addr_format_is_global(addr_format, modes)) { return build_explicit_io_atomic(b, intrin, addr, addr_format, nir_var_mem_global); } else if (modes & nir_var_function_temp) { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_function_temp)); nir_def *res1 = build_explicit_io_atomic(b, intrin, addr, addr_format, nir_var_function_temp); nir_push_else(b, NULL); nir_def *res2 = build_explicit_io_atomic(b, intrin, addr, addr_format, modes & ~nir_var_function_temp); nir_pop_if(b, NULL); return nir_if_phi(b, res1, res2); } else { nir_push_if(b, build_runtime_addr_mode_check(b, addr, addr_format, nir_var_mem_shared)); assert(modes & nir_var_mem_shared); nir_def *res1 = build_explicit_io_atomic(b, intrin, addr, addr_format, nir_var_mem_shared); nir_push_else(b, NULL); assert(modes & nir_var_mem_global); nir_def *res2 = build_explicit_io_atomic(b, intrin, addr, addr_format, nir_var_mem_global); nir_pop_if(b, NULL); return nir_if_phi(b, res1, res2); } } assert(util_bitcount(modes) == 1); const nir_variable_mode mode = modes; const unsigned num_data_srcs = nir_intrinsic_infos[intrin->intrinsic].num_srcs - 1; nir_intrinsic_op op; switch (mode) { case nir_var_mem_ssbo: if (addr_format_is_global(addr_format, mode)) op = global_atomic_for_deref(addr_format, intrin->intrinsic); else op = ssbo_atomic_for_deref(intrin->intrinsic); break; case nir_var_mem_global: assert(addr_format_is_global(addr_format, mode)); op = global_atomic_for_deref(addr_format, intrin->intrinsic); break; case nir_var_mem_shared: assert(addr_format_is_offset(addr_format, mode)); op = shared_atomic_for_deref(intrin->intrinsic); break; case nir_var_mem_task_payload: assert(addr_format_is_offset(addr_format, mode)); op = task_payload_atomic_for_deref(intrin->intrinsic); break; default: unreachable("Unsupported explicit IO variable mode"); } nir_intrinsic_instr *atomic = nir_intrinsic_instr_create(b->shader, op); nir_intrinsic_set_atomic_op(atomic, nir_intrinsic_atomic_op(intrin)); unsigned src = 0; if (addr_format_is_global(addr_format, mode)) { atomic->src[src++] = nir_src_for_ssa(addr_to_global(b, addr, addr_format)); } else if (addr_format_is_offset(addr_format, mode)) { assert(addr->num_components == 1); atomic->src[src++] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } else { atomic->src[src++] = nir_src_for_ssa(addr_to_index(b, addr, addr_format)); atomic->src[src++] = nir_src_for_ssa(addr_to_offset(b, addr, addr_format)); } for (unsigned i = 0; i < num_data_srcs; i++) { atomic->src[src++] = nir_src_for_ssa(intrin->src[1 + i].ssa); } /* Global atomics don't have access flags because they assume that the * address may be non-uniform. */ if (nir_intrinsic_has_access(atomic)) nir_intrinsic_set_access(atomic, nir_intrinsic_access(intrin)); assert(intrin->def.num_components == 1); nir_def_init(&atomic->instr, &atomic->def, 1, intrin->def.bit_size); assert(atomic->def.bit_size % 8 == 0); if (addr_format_needs_bounds_check(addr_format)) { const unsigned atomic_size = atomic->def.bit_size / 8; nir_push_if(b, addr_is_in_bounds(b, addr, addr_format, atomic_size)); nir_builder_instr_insert(b, &atomic->instr); nir_pop_if(b, NULL); return nir_if_phi(b, &atomic->def, nir_undef(b, 1, atomic->def.bit_size)); } else { nir_builder_instr_insert(b, &atomic->instr); return &atomic->def; } } nir_def * nir_explicit_io_address_from_deref(nir_builder *b, nir_deref_instr *deref, nir_def *base_addr, nir_address_format addr_format) { switch (deref->deref_type) { case nir_deref_type_var: return build_addr_for_var(b, deref->var, addr_format); case nir_deref_type_ptr_as_array: case nir_deref_type_array: { unsigned stride = nir_deref_instr_array_stride(deref); assert(stride > 0); unsigned offset_bit_size = addr_get_offset_bit_size(base_addr, addr_format); nir_def *index = deref->arr.index.ssa; nir_def *offset; /* If the access chain has been declared in-bounds, then we know it doesn't * overflow the type. For nir_deref_type_array, this implies it cannot be * negative. Also, since types in NIR have a maximum 32-bit size, we know the * final result will fit in a 32-bit value so we can convert the index to * 32-bit before multiplying and save ourselves from a 64-bit multiply. */ if (deref->arr.in_bounds && deref->deref_type == nir_deref_type_array) { index = nir_u2u32(b, index); offset = nir_u2uN(b, nir_amul_imm(b, index, stride), offset_bit_size); } else { index = nir_i2iN(b, index, offset_bit_size); offset = nir_amul_imm(b, index, stride); } return nir_build_addr_iadd(b, base_addr, addr_format, deref->modes, offset); } case nir_deref_type_array_wildcard: unreachable("Wildcards should be lowered by now"); break; case nir_deref_type_struct: { nir_deref_instr *parent = nir_deref_instr_parent(deref); int offset = glsl_get_struct_field_offset(parent->type, deref->strct.index); assert(offset >= 0); return nir_build_addr_iadd_imm(b, base_addr, addr_format, deref->modes, offset); } case nir_deref_type_cast: /* Nothing to do here */ return base_addr; } unreachable("Invalid NIR deref type"); } void nir_lower_explicit_io_instr(nir_builder *b, nir_intrinsic_instr *intrin, nir_def *addr, nir_address_format addr_format) { b->cursor = nir_after_instr(&intrin->instr); nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); unsigned vec_stride = glsl_get_explicit_stride(deref->type); unsigned scalar_size = type_scalar_size_bytes(deref->type); if (vec_stride == 0) { vec_stride = scalar_size; } else { assert(glsl_type_is_vector(deref->type)); assert(vec_stride >= scalar_size); } uint32_t align_mul, align_offset; if (!nir_get_explicit_deref_align(deref, true, &align_mul, &align_offset)) { /* If we don't have an alignment from the deref, assume scalar */ align_mul = scalar_size; align_offset = 0; } /* In order for bounds checking to be correct as per the Vulkan spec, * we need to check at the individual component granularity. Prior to * robustness2, we're technically allowed to be sloppy by 16B. Even with * robustness2, UBO loads are allowed to have a granularity as high as 256B * depending on hardware limits. However, we have none of that information * here. Short of adding new address formats, the easiest way to do that * is to just split any loads and stores into individual components here. * * TODO: At some point in the future we may want to add more ops similar to * nir_intrinsic_load_global_constant_bounded and make bouds checking the * back-end's problem. Another option would be to somehow plumb more of * that information through to nir_lower_explicit_io. For now, however, * scalarizing is at least correct. */ bool scalarize = vec_stride > scalar_size || addr_format_needs_bounds_check(addr_format); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: { nir_def *value; if (scalarize) { nir_def *comps[NIR_MAX_VEC_COMPONENTS] = { NULL, }; for (unsigned i = 0; i < intrin->num_components; i++) { unsigned comp_offset = i * vec_stride; nir_def *comp_addr = nir_build_addr_iadd_imm(b, addr, addr_format, deref->modes, comp_offset); comps[i] = build_explicit_io_load(b, intrin, comp_addr, addr_format, deref->modes, align_mul, (align_offset + comp_offset) % align_mul, 1); } value = nir_vec(b, comps, intrin->num_components); } else { value = build_explicit_io_load(b, intrin, addr, addr_format, deref->modes, align_mul, align_offset, intrin->num_components); } nir_def_rewrite_uses(&intrin->def, value); break; } case nir_intrinsic_store_deref: { nir_def *value = intrin->src[1].ssa; nir_component_mask_t write_mask = nir_intrinsic_write_mask(intrin); if (scalarize) { for (unsigned i = 0; i < intrin->num_components; i++) { if (!(write_mask & (1 << i))) continue; unsigned comp_offset = i * vec_stride; nir_def *comp_addr = nir_build_addr_iadd_imm(b, addr, addr_format, deref->modes, comp_offset); build_explicit_io_store(b, intrin, comp_addr, addr_format, deref->modes, align_mul, (align_offset + comp_offset) % align_mul, nir_channel(b, value, i), 1); } } else { build_explicit_io_store(b, intrin, addr, addr_format, deref->modes, align_mul, align_offset, value, write_mask); } break; } case nir_intrinsic_load_deref_block_intel: { nir_def *value = build_explicit_io_load(b, intrin, addr, addr_format, deref->modes, align_mul, align_offset, intrin->num_components); nir_def_rewrite_uses(&intrin->def, value); break; } case nir_intrinsic_store_deref_block_intel: { nir_def *value = intrin->src[1].ssa; const nir_component_mask_t write_mask = 0; build_explicit_io_store(b, intrin, addr, addr_format, deref->modes, align_mul, align_offset, value, write_mask); break; } default: { nir_def *value = build_explicit_io_atomic(b, intrin, addr, addr_format, deref->modes); nir_def_rewrite_uses(&intrin->def, value); break; } } nir_instr_remove(&intrin->instr); } bool nir_get_explicit_deref_align(nir_deref_instr *deref, bool default_to_type_align, uint32_t *align_mul, uint32_t *align_offset) { if (deref->deref_type == nir_deref_type_var) { /* If we see a variable, align_mul is effectively infinite because we * know the offset exactly (up to the offset of the base pointer for the * given variable mode). We have to pick something so we choose 256B * as an arbitrary alignment which seems high enough for any reasonable * wide-load use-case. Back-ends should clamp alignments down if 256B * is too large for some reason. */ *align_mul = 256; *align_offset = deref->var->data.driver_location % 256; return true; } /* If we're a cast deref that has an alignment, use that. */ if (deref->deref_type == nir_deref_type_cast && deref->cast.align_mul > 0) { *align_mul = deref->cast.align_mul; *align_offset = deref->cast.align_offset; return true; } /* Otherwise, we need to compute the alignment based on the parent */ nir_deref_instr *parent = nir_deref_instr_parent(deref); if (parent == NULL) { assert(deref->deref_type == nir_deref_type_cast); if (default_to_type_align) { /* If we don't have a parent, assume the type's alignment, if any. */ unsigned type_align = glsl_get_explicit_alignment(deref->type); if (type_align == 0) return false; *align_mul = type_align; *align_offset = 0; return true; } else { return false; } } uint32_t parent_mul, parent_offset; if (!nir_get_explicit_deref_align(parent, default_to_type_align, &parent_mul, &parent_offset)) return false; switch (deref->deref_type) { case nir_deref_type_var: unreachable("Handled above"); case nir_deref_type_array: case nir_deref_type_array_wildcard: case nir_deref_type_ptr_as_array: { const unsigned stride = nir_deref_instr_array_stride(deref); if (stride == 0) return false; if (deref->deref_type != nir_deref_type_array_wildcard && nir_src_is_const(deref->arr.index)) { unsigned offset = nir_src_as_uint(deref->arr.index) * stride; *align_mul = parent_mul; *align_offset = (parent_offset + offset) % parent_mul; } else { /* If this is a wildcard or an indirect deref, we have to go with the * power-of-two gcd. */ *align_mul = MIN2(parent_mul, 1 << (ffs(stride) - 1)); *align_offset = parent_offset % *align_mul; } return true; } case nir_deref_type_struct: { const int offset = glsl_get_struct_field_offset(parent->type, deref->strct.index); if (offset < 0) return false; *align_mul = parent_mul; *align_offset = (parent_offset + offset) % parent_mul; return true; } case nir_deref_type_cast: /* We handled the explicit alignment case above. */ assert(deref->cast.align_mul == 0); *align_mul = parent_mul; *align_offset = parent_offset; return true; } unreachable("Invalid deref_instr_type"); } static void lower_explicit_io_deref(nir_builder *b, nir_deref_instr *deref, nir_address_format addr_format) { /* Ignore samplers/textures, because they are handled by other passes like `nir_lower_samplers`. * Also do it only for those being uniforms, otherwise it will break GL bindless textures handles * stored in UBOs. */ if (nir_deref_mode_is_in_set(deref, nir_var_uniform) && (glsl_type_is_sampler(deref->type) || glsl_type_is_texture(deref->type))) return; /* Just delete the deref if it's not used. We can't use * nir_deref_instr_remove_if_unused here because it may remove more than * one deref which could break our list walking since we walk the list * backwards. */ if (nir_def_is_unused(&deref->def)) { nir_instr_remove(&deref->instr); return; } b->cursor = nir_after_instr(&deref->instr); nir_def *base_addr = NULL; if (deref->deref_type != nir_deref_type_var) { base_addr = deref->parent.ssa; } nir_def *addr = nir_explicit_io_address_from_deref(b, deref, base_addr, addr_format); assert(addr->bit_size == deref->def.bit_size); assert(addr->num_components == deref->def.num_components); nir_instr_remove(&deref->instr); nir_def_rewrite_uses(&deref->def, addr); } static void lower_explicit_io_access(nir_builder *b, nir_intrinsic_instr *intrin, nir_address_format addr_format) { nir_lower_explicit_io_instr(b, intrin, intrin->src[0].ssa, addr_format); } static void lower_explicit_io_array_length(nir_builder *b, nir_intrinsic_instr *intrin, nir_address_format addr_format) { b->cursor = nir_after_instr(&intrin->instr); nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); assert(glsl_type_is_array(deref->type)); assert(glsl_get_length(deref->type) == 0); assert(nir_deref_mode_is(deref, nir_var_mem_ssbo)); unsigned stride = glsl_get_explicit_stride(deref->type); assert(stride > 0); nir_def *addr = &deref->def; nir_def *offset, *size; switch (addr_format) { case nir_address_format_64bit_global_32bit_offset: case nir_address_format_64bit_bounded_global: offset = nir_channel(b, addr, 3); size = nir_channel(b, addr, 2); break; case nir_address_format_32bit_index_offset: case nir_address_format_32bit_index_offset_pack64: case nir_address_format_vec2_index_32bit_offset: { offset = addr_to_offset(b, addr, addr_format); nir_def *index = addr_to_index(b, addr, addr_format); unsigned access = nir_intrinsic_access(intrin); size = nir_get_ssbo_size(b, index, .access = access); break; } default: unreachable("Cannot determine SSBO size"); } nir_def *remaining = nir_usub_sat(b, size, offset); nir_def *arr_size = nir_udiv_imm(b, remaining, stride); nir_def_replace(&intrin->def, arr_size); } static void lower_explicit_io_mode_check(nir_builder *b, nir_intrinsic_instr *intrin, nir_address_format addr_format) { if (addr_format_is_global(addr_format, 0)) { /* If the address format is always global, then the driver can use * global addresses regardless of the mode. In that case, don't create * a check, just whack the intrinsic to addr_mode_is and delegate to the * driver lowering. */ intrin->intrinsic = nir_intrinsic_addr_mode_is; return; } nir_def *addr = intrin->src[0].ssa; b->cursor = nir_instr_remove(&intrin->instr); nir_def *is_mode = build_runtime_addr_mode_check(b, addr, addr_format, nir_intrinsic_memory_modes(intrin)); nir_def_rewrite_uses(&intrin->def, is_mode); } static bool nir_lower_explicit_io_impl(nir_function_impl *impl, nir_variable_mode modes, nir_address_format addr_format) { bool progress = false; nir_builder b = nir_builder_create(impl); /* Walk in reverse order so that we can see the full deref chain when we * lower the access operations. We lower them assuming that the derefs * will be turned into address calculations later. */ nir_foreach_block_reverse(block, impl) { nir_foreach_instr_reverse_safe(instr, block) { switch (instr->type) { case nir_instr_type_deref: { nir_deref_instr *deref = nir_instr_as_deref(instr); if (nir_deref_mode_is_in_set(deref, modes)) { lower_explicit_io_deref(&b, deref, addr_format); progress = true; } break; } case nir_instr_type_intrinsic: { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: case nir_intrinsic_store_deref: case nir_intrinsic_load_deref_block_intel: case nir_intrinsic_store_deref_block_intel: case nir_intrinsic_deref_atomic: case nir_intrinsic_deref_atomic_swap: { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (nir_deref_mode_is_in_set(deref, modes)) { lower_explicit_io_access(&b, intrin, addr_format); progress = true; } break; } case nir_intrinsic_deref_buffer_array_length: { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (nir_deref_mode_is_in_set(deref, modes)) { lower_explicit_io_array_length(&b, intrin, addr_format); progress = true; } break; } case nir_intrinsic_deref_mode_is: { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (nir_deref_mode_is_in_set(deref, modes)) { lower_explicit_io_mode_check(&b, intrin, addr_format); progress = true; } break; } case nir_intrinsic_launch_mesh_workgroups_with_payload_deref: { if (modes & nir_var_mem_task_payload) { /* Get address and size of the payload variable. */ nir_deref_instr *deref = nir_src_as_deref(intrin->src[1]); assert(deref->deref_type == nir_deref_type_var); unsigned base = deref->var->data.explicit_location; unsigned size = glsl_get_explicit_size(deref->var->type, false); /* Replace the current instruction with the explicit intrinsic. */ nir_def *dispatch_3d = intrin->src[0].ssa; b.cursor = nir_instr_remove(instr); nir_launch_mesh_workgroups(&b, dispatch_3d, .base = base, .range = size); progress = true; } break; } default: break; } break; } default: /* Nothing to do */ break; } } } if (progress) { nir_metadata_preserve(impl, nir_metadata_none); } else { nir_metadata_preserve(impl, nir_metadata_all); } return progress; } /** Lower explicitly laid out I/O access to byte offset/address intrinsics * * This pass is intended to be used for any I/O which touches memory external * to the shader or which is directly visible to the client. It requires that * all data types in the given modes have a explicit stride/offset decorations * to tell it exactly how to calculate the offset/address for the given load, * store, or atomic operation. If the offset/stride information does not come * from the client explicitly (as with shared variables in GL or Vulkan), * nir_lower_vars_to_explicit_types() can be used to add them. * * Unlike nir_lower_io, this pass is fully capable of handling incomplete * pointer chains which may contain cast derefs. It does so by walking the * deref chain backwards and simply replacing each deref, one at a time, with * the appropriate address calculation. The pass takes a nir_address_format * parameter which describes how the offset or address is to be represented * during calculations. By ensuring that the address is always in a * consistent format, pointers can safely be conjured from thin air by the * driver, stored to variables, passed through phis, etc. * * The one exception to the simple algorithm described above is for handling * row-major matrices in which case we may look down one additional level of * the deref chain. * * This pass is also capable of handling OpenCL generic pointers. If the * address mode is global, it will lower any ambiguous (more than one mode) * access to global and pass through the deref_mode_is run-time checks as * addr_mode_is. This assumes the driver has somehow mapped shared and * scratch memory to the global address space. For other modes such as * 62bit_generic, there is an enum embedded in the address and we lower * ambiguous access to an if-ladder and deref_mode_is to a check against the * embedded enum. If nir_lower_explicit_io is called on any shader that * contains generic pointers, it must either be used on all of the generic * modes or none. */ bool nir_lower_explicit_io(nir_shader *shader, nir_variable_mode modes, nir_address_format addr_format) { bool progress = false; nir_foreach_function_impl(impl, shader) { if (impl && nir_lower_explicit_io_impl(impl, modes, addr_format)) progress = true; } return progress; } static bool nir_lower_vars_to_explicit_types_impl(nir_function_impl *impl, nir_variable_mode modes, glsl_type_size_align_func type_info) { bool progress = false; nir_foreach_block(block, impl) { nir_foreach_instr(instr, block) { if (instr->type != nir_instr_type_deref) continue; nir_deref_instr *deref = nir_instr_as_deref(instr); if (!nir_deref_mode_is_in_set(deref, modes)) continue; unsigned size, alignment; const struct glsl_type *new_type = glsl_get_explicit_type_for_size_align(deref->type, type_info, &size, &alignment); if (new_type != deref->type) { progress = true; deref->type = new_type; } if (deref->deref_type == nir_deref_type_cast) { /* See also glsl_type::get_explicit_type_for_size_align() */ unsigned new_stride = align(size, alignment); if (new_stride != deref->cast.ptr_stride) { deref->cast.ptr_stride = new_stride; progress = true; } } } } if (progress) { nir_metadata_preserve(impl, nir_metadata_control_flow | nir_metadata_live_defs | nir_metadata_loop_analysis); } else { nir_metadata_preserve(impl, nir_metadata_all); } return progress; } static bool lower_vars_to_explicit(nir_shader *shader, struct exec_list *vars, nir_variable_mode mode, glsl_type_size_align_func type_info) { bool progress = false; unsigned offset; switch (mode) { case nir_var_uniform: assert(shader->info.stage == MESA_SHADER_KERNEL); offset = 0; break; case nir_var_function_temp: case nir_var_shader_temp: offset = shader->scratch_size; break; case nir_var_mem_shared: offset = shader->info.shared_size; break; case nir_var_mem_task_payload: offset = shader->info.task_payload_size; break; case nir_var_mem_node_payload: assert(!shader->info.cs.node_payloads_size); offset = 0; break; case nir_var_mem_global: offset = shader->global_mem_size; break; case nir_var_mem_constant: offset = shader->constant_data_size; break; case nir_var_shader_call_data: case nir_var_ray_hit_attrib: case nir_var_mem_node_payload_in: offset = 0; break; default: unreachable("Unsupported mode"); } nir_foreach_variable_in_list(var, vars) { if (var->data.mode != mode) continue; unsigned size, alignment; const struct glsl_type *explicit_type = glsl_get_explicit_type_for_size_align(var->type, type_info, &size, &alignment); if (explicit_type != var->type) var->type = explicit_type; UNUSED bool is_empty_struct = glsl_type_is_struct_or_ifc(explicit_type) && glsl_get_length(explicit_type) == 0; assert(util_is_power_of_two_nonzero(alignment) || is_empty_struct || glsl_type_is_cmat(glsl_without_array(explicit_type))); assert(util_is_power_of_two_or_zero(var->data.alignment)); alignment = MAX2(alignment, var->data.alignment); var->data.driver_location = ALIGN_POT(offset, alignment); offset = var->data.driver_location + size; progress = true; } switch (mode) { case nir_var_uniform: assert(shader->info.stage == MESA_SHADER_KERNEL); shader->num_uniforms = offset; break; case nir_var_shader_temp: case nir_var_function_temp: shader->scratch_size = offset; break; case nir_var_mem_shared: shader->info.shared_size = offset; break; case nir_var_mem_task_payload: shader->info.task_payload_size = offset; break; case nir_var_mem_node_payload: shader->info.cs.node_payloads_size = offset; break; case nir_var_mem_global: shader->global_mem_size = offset; break; case nir_var_mem_constant: shader->constant_data_size = offset; break; case nir_var_shader_call_data: case nir_var_ray_hit_attrib: case nir_var_mem_node_payload_in: break; default: unreachable("Unsupported mode"); } return progress; } /* If nir_lower_vars_to_explicit_types is called on any shader that contains * generic pointers, it must either be used on all of the generic modes or * none. */ bool nir_lower_vars_to_explicit_types(nir_shader *shader, nir_variable_mode modes, glsl_type_size_align_func type_info) { /* TODO: Situations which need to be handled to support more modes: * - row-major matrices * - compact shader inputs/outputs * - interface types */ ASSERTED nir_variable_mode supported = nir_var_mem_shared | nir_var_mem_global | nir_var_mem_constant | nir_var_shader_temp | nir_var_function_temp | nir_var_uniform | nir_var_shader_call_data | nir_var_ray_hit_attrib | nir_var_mem_task_payload | nir_var_mem_node_payload | nir_var_mem_node_payload_in; assert(!(modes & ~supported) && "unsupported"); bool progress = false; if (modes & nir_var_uniform) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_uniform, type_info); if (modes & nir_var_mem_global) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_global, type_info); if (modes & nir_var_mem_shared) { assert(!shader->info.shared_memory_explicit_layout); progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_shared, type_info); } if (modes & nir_var_shader_temp) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_shader_temp, type_info); if (modes & nir_var_mem_constant) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_constant, type_info); if (modes & nir_var_shader_call_data) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_shader_call_data, type_info); if (modes & nir_var_ray_hit_attrib) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_ray_hit_attrib, type_info); if (modes & nir_var_mem_task_payload) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_task_payload, type_info); if (modes & nir_var_mem_node_payload) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_node_payload, type_info); if (modes & nir_var_mem_node_payload_in) progress |= lower_vars_to_explicit(shader, &shader->variables, nir_var_mem_node_payload_in, type_info); nir_foreach_function_impl(impl, shader) { if (modes & nir_var_function_temp) progress |= lower_vars_to_explicit(shader, &impl->locals, nir_var_function_temp, type_info); progress |= nir_lower_vars_to_explicit_types_impl(impl, modes, type_info); } return progress; } static void write_constant(void *dst, size_t dst_size, const nir_constant *c, const struct glsl_type *type) { if (c->is_null_constant) { memset(dst, 0, dst_size); return; } if (glsl_type_is_vector_or_scalar(type)) { const unsigned num_components = glsl_get_vector_elements(type); const unsigned bit_size = glsl_get_bit_size(type); if (bit_size == 1) { /* Booleans are special-cased to be 32-bit * * TODO: Make the native bool bit_size an option. */ assert(num_components * 4 <= dst_size); for (unsigned i = 0; i < num_components; i++) { int32_t b32 = -(int)c->values[i].b; memcpy((char *)dst + i * 4, &b32, 4); } } else { assert(bit_size >= 8 && bit_size % 8 == 0); const unsigned byte_size = bit_size / 8; assert(num_components * byte_size <= dst_size); for (unsigned i = 0; i < num_components; i++) { /* Annoyingly, thanks to packed structs, we can't make any * assumptions about the alignment of dst. To avoid any strange * issues with unaligned writes, we always use memcpy. */ memcpy((char *)dst + i * byte_size, &c->values[i], byte_size); } } } else if (glsl_type_is_array_or_matrix(type)) { const unsigned array_len = glsl_get_length(type); const unsigned stride = glsl_get_explicit_stride(type); assert(stride > 0); const struct glsl_type *elem_type = glsl_get_array_element(type); for (unsigned i = 0; i < array_len; i++) { unsigned elem_offset = i * stride; assert(elem_offset < dst_size); write_constant((char *)dst + elem_offset, dst_size - elem_offset, c->elements[i], elem_type); } } else { assert(glsl_type_is_struct_or_ifc(type)); const unsigned num_fields = glsl_get_length(type); for (unsigned i = 0; i < num_fields; i++) { const int field_offset = glsl_get_struct_field_offset(type, i); assert(field_offset >= 0 && field_offset < dst_size); const struct glsl_type *field_type = glsl_get_struct_field(type, i); write_constant((char *)dst + field_offset, dst_size - field_offset, c->elements[i], field_type); } } } void nir_gather_explicit_io_initializers(nir_shader *shader, void *dst, size_t dst_size, nir_variable_mode mode) { /* It doesn't really make sense to gather initializers for more than one * mode at a time. If this ever becomes well-defined, we can drop the * assert then. */ assert(util_bitcount(mode) == 1); nir_foreach_variable_with_modes(var, shader, mode) { assert(var->data.driver_location < dst_size); write_constant((char *)dst + var->data.driver_location, dst_size - var->data.driver_location, var->constant_initializer, var->type); } } /** * Return the offset source number for a load/store intrinsic or -1 if there's no offset. */ int nir_get_io_offset_src_number(const nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_input: case nir_intrinsic_load_per_primitive_input: case nir_intrinsic_load_output: case nir_intrinsic_load_shared: case nir_intrinsic_load_task_payload: case nir_intrinsic_load_uniform: case nir_intrinsic_load_push_constant: case nir_intrinsic_load_kernel_input: case nir_intrinsic_load_global: case nir_intrinsic_load_global_2x32: case nir_intrinsic_load_global_constant: case nir_intrinsic_load_global_etna: case nir_intrinsic_load_scratch: case nir_intrinsic_load_fs_input_interp_deltas: case nir_intrinsic_shared_atomic: case nir_intrinsic_shared_atomic_swap: case nir_intrinsic_task_payload_atomic: case nir_intrinsic_task_payload_atomic_swap: case nir_intrinsic_global_atomic: case nir_intrinsic_global_atomic_2x32: case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_swap_2x32: case nir_intrinsic_load_coefficients_agx: return 0; case nir_intrinsic_load_ubo: case nir_intrinsic_load_ssbo: case nir_intrinsic_load_input_vertex: case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_load_per_primitive_output: case nir_intrinsic_load_interpolated_input: case nir_intrinsic_store_output: case nir_intrinsic_store_shared: case nir_intrinsic_store_task_payload: case nir_intrinsic_store_global: case nir_intrinsic_store_global_2x32: case nir_intrinsic_store_global_etna: case nir_intrinsic_store_scratch: case nir_intrinsic_ssbo_atomic: case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ldc_nv: case nir_intrinsic_ldcx_nv: return 1; case nir_intrinsic_store_ssbo: case nir_intrinsic_store_per_vertex_output: case nir_intrinsic_store_per_primitive_output: return 2; default: return -1; } } /** * Return the offset source for a load/store intrinsic. */ nir_src * nir_get_io_offset_src(nir_intrinsic_instr *instr) { const int idx = nir_get_io_offset_src_number(instr); return idx >= 0 ? &instr->src[idx] : NULL; } /** * Return the vertex index source number for a load/store per_vertex intrinsic or -1 if there's no offset. */ int nir_get_io_arrayed_index_src_number(const nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_load_per_primitive_output: return 0; case nir_intrinsic_store_per_vertex_output: case nir_intrinsic_store_per_primitive_output: return 1; default: return -1; } } /** * Return the vertex index source for a load/store per_vertex intrinsic. */ nir_src * nir_get_io_arrayed_index_src(nir_intrinsic_instr *instr) { const int idx = nir_get_io_arrayed_index_src_number(instr); return idx >= 0 ? &instr->src[idx] : NULL; } /** * Return the numeric constant that identify a NULL pointer for each address * format. */ const nir_const_value * nir_address_format_null_value(nir_address_format addr_format) { const static nir_const_value null_values[][NIR_MAX_VEC_COMPONENTS] = { [nir_address_format_32bit_global] = { { 0 } }, [nir_address_format_2x32bit_global] = { { 0 } }, [nir_address_format_64bit_global] = { { 0 } }, [nir_address_format_64bit_global_32bit_offset] = { { 0 } }, [nir_address_format_64bit_bounded_global] = { { 0 } }, [nir_address_format_32bit_index_offset] = { { .u32 = ~0 }, { .u32 = ~0 } }, [nir_address_format_32bit_index_offset_pack64] = { { .u64 = ~0ull } }, [nir_address_format_vec2_index_32bit_offset] = { { .u32 = ~0 }, { .u32 = ~0 }, { .u32 = ~0 } }, [nir_address_format_32bit_offset] = { { .u32 = ~0 } }, [nir_address_format_32bit_offset_as_64bit] = { { .u64 = ~0ull } }, [nir_address_format_62bit_generic] = { { .u64 = 0 } }, [nir_address_format_logical] = { { .u32 = ~0 } }, }; assert(addr_format < ARRAY_SIZE(null_values)); return null_values[addr_format]; } nir_def * nir_build_addr_ieq(nir_builder *b, nir_def *addr0, nir_def *addr1, nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_global: case nir_address_format_2x32bit_global: case nir_address_format_64bit_global: case nir_address_format_64bit_bounded_global: case nir_address_format_32bit_index_offset: case nir_address_format_vec2_index_32bit_offset: case nir_address_format_32bit_offset: case nir_address_format_62bit_generic: return nir_ball_iequal(b, addr0, addr1); case nir_address_format_64bit_global_32bit_offset: return nir_ball_iequal(b, nir_channels(b, addr0, 0xb), nir_channels(b, addr1, 0xb)); case nir_address_format_32bit_offset_as_64bit: assert(addr0->num_components == 1 && addr1->num_components == 1); return nir_ieq(b, nir_u2u32(b, addr0), nir_u2u32(b, addr1)); case nir_address_format_32bit_index_offset_pack64: assert(addr0->num_components == 1 && addr1->num_components == 1); return nir_ball_iequal(b, nir_unpack_64_2x32(b, addr0), nir_unpack_64_2x32(b, addr1)); case nir_address_format_logical: unreachable("Unsupported address format"); } unreachable("Invalid address format"); } nir_def * nir_build_addr_isub(nir_builder *b, nir_def *addr0, nir_def *addr1, nir_address_format addr_format) { switch (addr_format) { case nir_address_format_32bit_global: case nir_address_format_64bit_global: case nir_address_format_32bit_offset: case nir_address_format_32bit_index_offset_pack64: case nir_address_format_62bit_generic: assert(addr0->num_components == 1); assert(addr1->num_components == 1); return nir_isub(b, addr0, addr1); case nir_address_format_2x32bit_global: return nir_isub(b, addr_to_global(b, addr0, addr_format), addr_to_global(b, addr1, addr_format)); case nir_address_format_32bit_offset_as_64bit: assert(addr0->num_components == 1); assert(addr1->num_components == 1); return nir_u2u64(b, nir_isub(b, nir_u2u32(b, addr0), nir_u2u32(b, addr1))); case nir_address_format_64bit_global_32bit_offset: case nir_address_format_64bit_bounded_global: return nir_isub(b, addr_to_global(b, addr0, addr_format), addr_to_global(b, addr1, addr_format)); case nir_address_format_32bit_index_offset: assert(addr0->num_components == 2); assert(addr1->num_components == 2); /* Assume the same buffer index. */ return nir_isub(b, nir_channel(b, addr0, 1), nir_channel(b, addr1, 1)); case nir_address_format_vec2_index_32bit_offset: assert(addr0->num_components == 3); assert(addr1->num_components == 3); /* Assume the same buffer index. */ return nir_isub(b, nir_channel(b, addr0, 2), nir_channel(b, addr1, 2)); case nir_address_format_logical: unreachable("Unsupported address format"); } unreachable("Invalid address format"); } static bool is_input(nir_intrinsic_instr *intrin) { return intrin->intrinsic == nir_intrinsic_load_input || intrin->intrinsic == nir_intrinsic_load_per_primitive_input || intrin->intrinsic == nir_intrinsic_load_input_vertex || intrin->intrinsic == nir_intrinsic_load_per_vertex_input || intrin->intrinsic == nir_intrinsic_load_interpolated_input || intrin->intrinsic == nir_intrinsic_load_fs_input_interp_deltas; } static bool is_output(nir_intrinsic_instr *intrin) { return intrin->intrinsic == nir_intrinsic_load_output || intrin->intrinsic == nir_intrinsic_load_per_vertex_output || intrin->intrinsic == nir_intrinsic_load_per_primitive_output || intrin->intrinsic == nir_intrinsic_store_output || intrin->intrinsic == nir_intrinsic_store_per_vertex_output || intrin->intrinsic == nir_intrinsic_store_per_primitive_output; } static bool is_dual_slot(nir_intrinsic_instr *intrin) { if (intrin->intrinsic == nir_intrinsic_store_output || intrin->intrinsic == nir_intrinsic_store_per_vertex_output || intrin->intrinsic == nir_intrinsic_store_per_primitive_output) { return nir_src_bit_size(intrin->src[0]) == 64 && nir_src_num_components(intrin->src[0]) >= 3; } return intrin->def.bit_size == 64 && intrin->def.num_components >= 3; } /** * This pass adds constant offsets to instr->const_index[0] for input/output * intrinsics, and resets the offset source to 0. Non-constant offsets remain * unchanged - since we don't know what part of a compound variable is * accessed, we allocate storage for the entire thing. For drivers that use * nir_lower_io_to_temporaries() before nir_lower_io(), this guarantees that * the offset source will be 0, so that they don't have to add it in manually. */ static bool add_const_offset_to_base_block(nir_block *block, nir_builder *b, nir_variable_mode modes) { bool progress = false; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (((modes & nir_var_shader_in) && is_input(intrin)) || ((modes & nir_var_shader_out) && is_output(intrin))) { nir_io_semantics sem = nir_intrinsic_io_semantics(intrin); /* NV_mesh_shader: ignore MS primitive indices. */ if (b->shader->info.stage == MESA_SHADER_MESH && sem.location == VARYING_SLOT_PRIMITIVE_INDICES && !(b->shader->info.per_primitive_outputs & BITFIELD64_BIT(VARYING_SLOT_PRIMITIVE_INDICES))) continue; nir_src *offset = nir_get_io_offset_src(intrin); /* TODO: Better handling of per-view variables here */ if (nir_src_is_const(*offset) && !nir_intrinsic_io_semantics(intrin).per_view) { unsigned off = nir_src_as_uint(*offset); nir_intrinsic_set_base(intrin, nir_intrinsic_base(intrin) + off); sem.location += off; /* non-indirect indexing should reduce num_slots */ sem.num_slots = is_dual_slot(intrin) ? 2 : 1; nir_intrinsic_set_io_semantics(intrin, sem); b->cursor = nir_before_instr(&intrin->instr); nir_src_rewrite(offset, nir_imm_int(b, 0)); progress = true; } } } return progress; } bool nir_io_add_const_offset_to_base(nir_shader *nir, nir_variable_mode modes) { bool progress = false; nir_foreach_function_impl(impl, nir) { bool impl_progress = false; nir_builder b = nir_builder_create(impl); nir_foreach_block(block, impl) { impl_progress |= add_const_offset_to_base_block(block, &b, modes); } progress |= impl_progress; if (impl_progress) nir_metadata_preserve(impl, nir_metadata_control_flow); else nir_metadata_preserve(impl, nir_metadata_all); } return progress; } bool nir_lower_color_inputs(nir_shader *nir) { nir_function_impl *impl = nir_shader_get_entrypoint(nir); bool progress = false; nir_builder b = nir_builder_create(impl); nir_foreach_block(block, impl) { nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (intrin->intrinsic != nir_intrinsic_load_input && intrin->intrinsic != nir_intrinsic_load_interpolated_input) continue; nir_io_semantics sem = nir_intrinsic_io_semantics(intrin); if (sem.location != VARYING_SLOT_COL0 && sem.location != VARYING_SLOT_COL1) continue; /* Default to FLAT (for load_input) */ enum glsl_interp_mode interp = INTERP_MODE_FLAT; bool sample = false; bool centroid = false; if (intrin->intrinsic == nir_intrinsic_load_interpolated_input) { nir_intrinsic_instr *baryc = nir_instr_as_intrinsic(intrin->src[0].ssa->parent_instr); centroid = baryc->intrinsic == nir_intrinsic_load_barycentric_centroid; sample = baryc->intrinsic == nir_intrinsic_load_barycentric_sample; assert(centroid || sample || baryc->intrinsic == nir_intrinsic_load_barycentric_pixel); interp = nir_intrinsic_interp_mode(baryc); } b.cursor = nir_before_instr(instr); nir_def *load = NULL; if (sem.location == VARYING_SLOT_COL0) { load = nir_load_color0(&b); nir->info.fs.color0_interp = interp; nir->info.fs.color0_sample = sample; nir->info.fs.color0_centroid = centroid; } else { assert(sem.location == VARYING_SLOT_COL1); load = nir_load_color1(&b); nir->info.fs.color1_interp = interp; nir->info.fs.color1_sample = sample; nir->info.fs.color1_centroid = centroid; } if (intrin->num_components != 4) { unsigned start = nir_intrinsic_component(intrin); unsigned count = intrin->num_components; load = nir_channels(&b, load, BITFIELD_RANGE(start, count)); } nir_def_replace(&intrin->def, load); progress = true; } } if (progress) { nir_metadata_preserve(impl, nir_metadata_control_flow); } else { nir_metadata_preserve(impl, nir_metadata_all); } return progress; } bool nir_io_add_intrinsic_xfb_info(nir_shader *nir) { nir_function_impl *impl = nir_shader_get_entrypoint(nir); bool progress = false; for (unsigned i = 0; i < NIR_MAX_XFB_BUFFERS; i++) nir->info.xfb_stride[i] = nir->xfb_info->buffers[i].stride / 4; nir_foreach_block(block, impl) { nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); if (!nir_intrinsic_has_io_xfb(intr)) continue; /* No indirect indexing allowed. The index is implied to be 0. */ ASSERTED nir_src offset = *nir_get_io_offset_src(intr); assert(nir_src_is_const(offset) && nir_src_as_uint(offset) == 0); /* Calling this pass for the second time shouldn't do anything. */ if (nir_intrinsic_io_xfb(intr).out[0].num_components || nir_intrinsic_io_xfb(intr).out[1].num_components || nir_intrinsic_io_xfb2(intr).out[0].num_components || nir_intrinsic_io_xfb2(intr).out[1].num_components) continue; nir_io_semantics sem = nir_intrinsic_io_semantics(intr); unsigned writemask = nir_intrinsic_write_mask(intr) << nir_intrinsic_component(intr); nir_io_xfb xfb[2]; memset(xfb, 0, sizeof(xfb)); for (unsigned i = 0; i < nir->xfb_info->output_count; i++) { nir_xfb_output_info *out = &nir->xfb_info->outputs[i]; if (out->location == sem.location) { unsigned xfb_mask = writemask & out->component_mask; /*fprintf(stdout, "output%u: buffer=%u, offset=%u, location=%u, " "component_offset=%u, component_mask=0x%x, xfb_mask=0x%x, slots=%u\n", i, out->buffer, out->offset, out->location, out->component_offset, out->component_mask, xfb_mask, sem.num_slots);*/ while (xfb_mask) { int start, count; u_bit_scan_consecutive_range(&xfb_mask, &start, &count); xfb[start / 2].out[start % 2].num_components = count; xfb[start / 2].out[start % 2].buffer = out->buffer; /* out->offset is relative to the first stored xfb component */ /* start is relative to component 0 */ xfb[start / 2].out[start % 2].offset = out->offset / 4 - out->component_offset + start; progress = true; } } } nir_intrinsic_set_io_xfb(intr, xfb[0]); nir_intrinsic_set_io_xfb2(intr, xfb[1]); } } nir_metadata_preserve(impl, nir_metadata_all); return progress; } static int type_size_vec4(const struct glsl_type *type, bool bindless) { return glsl_count_attribute_slots(type, false); } /** * This runs all compiler passes needed to lower IO, lower indirect IO access, * set transform feedback info in IO intrinsics, and clean up the IR. * * \param renumber_vs_inputs * Set to true if holes between VS inputs should be removed, which is safe * to do in any shader linker that can handle that. Set to false if you want * to keep holes between VS inputs, which is recommended to do in gallium * drivers so as not to break the mapping of vertex elements to VS inputs * expected by gallium frontends. */ void nir_lower_io_passes(nir_shader *nir, bool renumber_vs_inputs) { if (nir->info.stage == MESA_SHADER_COMPUTE) return; bool has_indirect_inputs = (nir->options->support_indirect_inputs >> nir->info.stage) & 0x1; /* Transform feedback requires that indirect outputs are lowered. */ bool has_indirect_outputs = (nir->options->support_indirect_outputs >> nir->info.stage) & 0x1 && nir->xfb_info == NULL; /* TODO: Sorting variables by location is required due to some bug * in nir_lower_io_to_temporaries. If variables are not sorted, * dEQP-GLES31.functional.separate_shader.random.0 fails. * * This isn't needed if nir_assign_io_var_locations is called because it * also sorts variables. However, if IO is lowered sooner than that, we * must sort explicitly here to get what nir_assign_io_var_locations does. */ unsigned varying_var_mask = (nir->info.stage != MESA_SHADER_VERTEX ? nir_var_shader_in : 0) | (nir->info.stage != MESA_SHADER_FRAGMENT ? nir_var_shader_out : 0); nir_sort_variables_by_location(nir, varying_var_mask); if (!has_indirect_inputs || !has_indirect_outputs) { NIR_PASS_V(nir, nir_lower_io_to_temporaries, nir_shader_get_entrypoint(nir), !has_indirect_outputs, !has_indirect_inputs); /* We need to lower all the copy_deref's introduced by lower_io_to- * _temporaries before calling nir_lower_io. */ NIR_PASS_V(nir, nir_split_var_copies); NIR_PASS_V(nir, nir_lower_var_copies); NIR_PASS_V(nir, nir_lower_global_vars_to_local); } /* The correct lower_64bit_to_32 flag is required by st/mesa depending * on whether the GLSL linker lowers IO or not. Setting the wrong flag * would break 64-bit vertex attribs for GLSL. */ NIR_PASS_V(nir, nir_lower_io, nir_var_shader_out | nir_var_shader_in, type_size_vec4, renumber_vs_inputs ? nir_lower_io_lower_64bit_to_32_new : nir_lower_io_lower_64bit_to_32); /* nir_io_add_const_offset_to_base needs actual constants. */ NIR_PASS_V(nir, nir_opt_constant_folding); NIR_PASS_V(nir, nir_io_add_const_offset_to_base, nir_var_shader_in | nir_var_shader_out); /* Lower and remove dead derefs and variables to clean up the IR. */ NIR_PASS_V(nir, nir_lower_vars_to_ssa); NIR_PASS_V(nir, nir_opt_dce); NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_function_temp, NULL); /* If IO is lowered before var->data.driver_location is assigned, driver * locations are all 0, which means IO bases are all 0. It's not necessary * to set driver_location before lowering IO because the only thing that * identifies outputs is their semantic, and IO bases can always be * computed from the semantics. * * This assigns IO bases from scratch, using IO semantics to tell which * intrinsics refer to the same IO. If the bases already exist, they * will be reassigned, sorted by the semantic, and all holes removed. * This kind of canonicalizes all bases. * * This must be done after DCE to remove dead load_input intrinsics. */ NIR_PASS_V(nir, nir_recompute_io_bases, (nir->info.stage != MESA_SHADER_VERTEX || renumber_vs_inputs ? nir_var_shader_in : 0) | nir_var_shader_out); if (nir->xfb_info) NIR_PASS_V(nir, nir_io_add_intrinsic_xfb_info); if (nir->options->lower_mediump_io) nir->options->lower_mediump_io(nir); nir->info.io_lowered = true; }