/* * Copyright © 2012 Intel Corporation * Copyright © 2021 Valve Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ /** * Linker functions related specifically to linking varyings between shader * stages. */ #include "main/errors.h" #include "main/macros.h" #include "main/menums.h" #include "main/mtypes.h" #include "program/symbol_table.h" #include "util/hash_table.h" #include "util/u_math.h" #include "util/perf/cpu_trace.h" #include "nir.h" #include "nir_builder.h" #include "nir_deref.h" #include "gl_nir.h" #include "gl_nir_link_varyings.h" #include "gl_nir_linker.h" #include "linker_util.h" #include "string_to_uint_map.h" #define SAFE_MASK_FROM_INDEX(i) (((i) >= 32) ? ~0 : ((1 << (i)) - 1)) /* Temporary storage for the set of attributes that need locations assigned. */ struct temp_attr { unsigned slots; unsigned original_idx; nir_variable *var; }; /* Used below in the call to qsort. */ static int compare_attr(const void *a, const void *b) { const struct temp_attr *const l = (const struct temp_attr *) a; const struct temp_attr *const r = (const struct temp_attr *) b; /* Reversed because we want a descending order sort below. */ if (r->slots != l->slots) return r->slots - l->slots; return l->original_idx - r->original_idx; } /** * Get the varying type stripped of the outermost array if we're processing * a stage whose varyings are arrays indexed by a vertex number (such as * geometry shader inputs). */ static const struct glsl_type * get_varying_type(const nir_variable *var, gl_shader_stage stage) { const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, stage) || var->data.per_view) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } return type; } /** * Find a contiguous set of available bits in a bitmask. * * \param used_mask Bits representing used (1) and unused (0) locations * \param needed_count Number of contiguous bits needed. * * \return * Base location of the available bits on success or -1 on failure. */ static int find_available_slots(unsigned used_mask, unsigned needed_count) { unsigned needed_mask = (1 << needed_count) - 1; const int max_bit_to_test = (8 * sizeof(used_mask)) - needed_count; /* The comparison to 32 is redundant, but without it GCC emits "warning: * cannot optimize possibly infinite loops" for the loop below. */ if ((needed_count == 0) || (max_bit_to_test < 0) || (max_bit_to_test > 32)) return -1; for (int i = 0; i <= max_bit_to_test; i++) { if ((needed_mask & ~used_mask) == needed_mask) return i; needed_mask <<= 1; } return -1; } /* Find deref based on variable name. * Note: This function does not support arrays. */ static bool find_deref(nir_shader *shader, const char *name) { nir_foreach_function(func, shader) { nir_foreach_block(block, func->impl) { nir_foreach_instr(instr, block) { if (instr->type == nir_instr_type_deref) { nir_deref_instr *deref = nir_instr_as_deref(instr); if (deref->deref_type == nir_deref_type_var && strcmp(deref->var->name, name) == 0) return true; } } } } return false; } /** * Validate the types and qualifiers of an output from one stage against the * matching input to another stage. */ static void cross_validate_types_and_qualifiers(const struct gl_constants *consts, struct gl_shader_program *prog, const nir_variable *input, const nir_variable *output, gl_shader_stage consumer_stage, gl_shader_stage producer_stage) { /* Check that the types match between stages. */ const struct glsl_type *type_to_match = input->type; /* VS -> GS, VS -> TCS, VS -> TES, TES -> GS */ const bool extra_array_level = (producer_stage == MESA_SHADER_VERTEX && consumer_stage != MESA_SHADER_FRAGMENT) || consumer_stage == MESA_SHADER_GEOMETRY; if (extra_array_level) { assert(glsl_type_is_array(type_to_match)); type_to_match = glsl_get_array_element(type_to_match); } if (type_to_match != output->type) { if (glsl_type_is_struct(output->type)) { /* Structures across shader stages can have different name * and considered to match in type if and only if structure * members match in name, type, qualification, and declaration * order. The precision doesn’t need to match. */ if (!glsl_record_compare(output->type, type_to_match, false, /* match_name */ true, /* match_locations */ false /* match_precision */)) { linker_error(prog, "%s shader output `%s' declared as struct `%s', " "doesn't match in type with %s shader input " "declared as struct `%s'\n", _mesa_shader_stage_to_string(producer_stage), output->name, glsl_get_type_name(output->type), _mesa_shader_stage_to_string(consumer_stage), glsl_get_type_name(input->type)); } } else if (!glsl_type_is_array(output->type) || !is_gl_identifier(output->name)) { /* There is a bit of a special case for gl_TexCoord. This * built-in is unsized by default. Applications that variable * access it must redeclare it with a size. There is some * language in the GLSL spec that implies the fragment shader * and vertex shader do not have to agree on this size. Other * driver behave this way, and one or two applications seem to * rely on it. * * Neither declaration needs to be modified here because the array * sizes are fixed later when update_array_sizes is called. * * From page 48 (page 54 of the PDF) of the GLSL 1.10 spec: * * "Unlike user-defined varying variables, the built-in * varying variables don't have a strict one-to-one * correspondence between the vertex language and the * fragment language." */ linker_error(prog, "%s shader output `%s' declared as type `%s', " "but %s shader input declared as type `%s'\n", _mesa_shader_stage_to_string(producer_stage), output->name, glsl_get_type_name(output->type), _mesa_shader_stage_to_string(consumer_stage), glsl_get_type_name(input->type)); return; } } /* Check that all of the qualifiers match between stages. */ /* According to the OpenGL and OpenGLES GLSL specs, the centroid qualifier * should match until OpenGL 4.3 and OpenGLES 3.1. The OpenGLES 3.0 * conformance test suite does not verify that the qualifiers must match. * The deqp test suite expects the opposite (OpenGLES 3.1) behavior for * OpenGLES 3.0 drivers, so we relax the checking in all cases. */ if (false /* always skip the centroid check */ && prog->GLSL_Version < (prog->IsES ? 310 : 430) && input->data.centroid != output->data.centroid) { linker_error(prog, "%s shader output `%s' %s centroid qualifier, " "but %s shader input %s centroid qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, (output->data.centroid) ? "has" : "lacks", _mesa_shader_stage_to_string(consumer_stage), (input->data.centroid) ? "has" : "lacks"); return; } if (input->data.sample != output->data.sample) { linker_error(prog, "%s shader output `%s' %s sample qualifier, " "but %s shader input %s sample qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, (output->data.sample) ? "has" : "lacks", _mesa_shader_stage_to_string(consumer_stage), (input->data.sample) ? "has" : "lacks"); return; } if (input->data.patch != output->data.patch) { linker_error(prog, "%s shader output `%s' %s patch qualifier, " "but %s shader input %s patch qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, (output->data.patch) ? "has" : "lacks", _mesa_shader_stage_to_string(consumer_stage), (input->data.patch) ? "has" : "lacks"); return; } /* The GLSL 4.20 and GLSL ES 3.00 specifications say: * * "As only outputs need be declared with invariant, an output from * one shader stage will still match an input of a subsequent stage * without the input being declared as invariant." * * while GLSL 4.10 says: * * "For variables leaving one shader and coming into another shader, * the invariant keyword has to be used in both shaders, or a link * error will result." * * and GLSL ES 1.00 section 4.6.4 "Invariance and Linking" says: * * "The invariance of varyings that are declared in both the vertex * and fragment shaders must match." */ if (input->data.explicit_invariant != output->data.explicit_invariant && prog->GLSL_Version < (prog->IsES ? 300 : 420)) { linker_error(prog, "%s shader output `%s' %s invariant qualifier, " "but %s shader input %s invariant qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, (output->data.explicit_invariant) ? "has" : "lacks", _mesa_shader_stage_to_string(consumer_stage), (input->data.explicit_invariant) ? "has" : "lacks"); return; } /* GLSL >= 4.40 removes text requiring interpolation qualifiers * to match cross stage, they must only match within the same stage. * * From page 84 (page 90 of the PDF) of the GLSL 4.40 spec: * * "It is a link-time error if, within the same stage, the interpolation * qualifiers of variables of the same name do not match. * * Section 4.3.9 (Interpolation) of the GLSL ES 3.00 spec says: * * "When no interpolation qualifier is present, smooth interpolation * is used." * * So we match variables where one is smooth and the other has no explicit * qualifier. */ unsigned input_interpolation = input->data.interpolation; unsigned output_interpolation = output->data.interpolation; if (prog->IsES) { if (input_interpolation == INTERP_MODE_NONE) input_interpolation = INTERP_MODE_SMOOTH; if (output_interpolation == INTERP_MODE_NONE) output_interpolation = INTERP_MODE_SMOOTH; } if (input_interpolation != output_interpolation && prog->GLSL_Version < 440) { if (!consts->AllowGLSLCrossStageInterpolationMismatch) { linker_error(prog, "%s shader output `%s' specifies %s " "interpolation qualifier, " "but %s shader input specifies %s " "interpolation qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, interpolation_string(output->data.interpolation), _mesa_shader_stage_to_string(consumer_stage), interpolation_string(input->data.interpolation)); return; } else { linker_warning(prog, "%s shader output `%s' specifies %s " "interpolation qualifier, " "but %s shader input specifies %s " "interpolation qualifier\n", _mesa_shader_stage_to_string(producer_stage), output->name, interpolation_string(output->data.interpolation), _mesa_shader_stage_to_string(consumer_stage), interpolation_string(input->data.interpolation)); } } } /** * Validate front and back color outputs against single color input */ static void cross_validate_front_and_back_color(const struct gl_constants *consts, struct gl_shader_program *prog, const nir_variable *input, const nir_variable *front_color, const nir_variable *back_color, gl_shader_stage consumer_stage, gl_shader_stage producer_stage) { if (front_color != NULL && front_color->data.assigned) cross_validate_types_and_qualifiers(consts, prog, input, front_color, consumer_stage, producer_stage); if (back_color != NULL && back_color->data.assigned) cross_validate_types_and_qualifiers(consts, prog, input, back_color, consumer_stage, producer_stage); } static unsigned compute_variable_location_slot(nir_variable *var, gl_shader_stage stage) { unsigned location_start = VARYING_SLOT_VAR0; switch (stage) { case MESA_SHADER_VERTEX: if (var->data.mode == nir_var_shader_in) location_start = VERT_ATTRIB_GENERIC0; break; case MESA_SHADER_TESS_CTRL: case MESA_SHADER_TESS_EVAL: if (var->data.patch) location_start = VARYING_SLOT_PATCH0; break; case MESA_SHADER_FRAGMENT: if (var->data.mode == nir_var_shader_out) location_start = FRAG_RESULT_DATA0; break; default: break; } return var->data.location - location_start; } struct explicit_location_info { nir_variable *var; bool base_type_is_integer; unsigned base_type_bit_size; unsigned interpolation; bool centroid; bool sample; bool patch; }; static bool check_location_aliasing(struct explicit_location_info explicit_locations[][4], nir_variable *var, unsigned location, unsigned component, unsigned location_limit, const struct glsl_type *type, unsigned interpolation, bool centroid, bool sample, bool patch, struct gl_shader_program *prog, gl_shader_stage stage) { unsigned last_comp; unsigned base_type_bit_size; const struct glsl_type *type_without_array = glsl_without_array(type); const bool base_type_is_integer = glsl_base_type_is_integer(glsl_get_base_type(type_without_array)); const bool is_struct = glsl_type_is_struct(type_without_array); if (is_struct) { /* structs don't have a defined underlying base type so just treat all * component slots as used and set the bit size to 0. If there is * location aliasing, we'll fail anyway later. */ last_comp = 4; base_type_bit_size = 0; } else { unsigned dmul = glsl_type_is_64bit(type_without_array) ? 2 : 1; last_comp = component + glsl_get_vector_elements(type_without_array) * dmul; base_type_bit_size = glsl_base_type_get_bit_size(glsl_get_base_type(type_without_array)); } while (location < location_limit) { unsigned comp = 0; while (comp < 4) { struct explicit_location_info *info = &explicit_locations[location][comp]; if (info->var) { if (glsl_type_is_struct(glsl_without_array(info->var->type)) || is_struct) { /* Structs cannot share location since they are incompatible * with any other underlying numerical type. */ linker_error(prog, "%s shader has multiple %sputs sharing the " "same location that don't have the same " "underlying numerical type. Struct variable '%s', " "location %u\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", is_struct ? var->name : info->var->name, location); return false; } else if (comp >= component && comp < last_comp) { /* Component aliasing is not allowed */ linker_error(prog, "%s shader has multiple %sputs explicitly " "assigned to location %d and component %d\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", location, comp); return false; } else { /* From the OpenGL 4.60.5 spec, section 4.4.1 Input Layout * Qualifiers, Page 67, (Location aliasing): * * " Further, when location aliasing, the aliases sharing the * location must have the same underlying numerical type * and bit width (floating-point or integer, 32-bit versus * 64-bit, etc.) and the same auxiliary storage and * interpolation qualification." */ /* If the underlying numerical type isn't integer, implicitly * it will be float or else we would have failed by now. */ if (info->base_type_is_integer != base_type_is_integer) { linker_error(prog, "%s shader has multiple %sputs sharing the " "same location that don't have the same " "underlying numerical type. Location %u " "component %u.\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", location, comp); return false; } if (info->base_type_bit_size != base_type_bit_size) { linker_error(prog, "%s shader has multiple %sputs sharing the " "same location that don't have the same " "underlying numerical bit size. Location %u " "component %u.\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", location, comp); return false; } if (info->interpolation != interpolation) { linker_error(prog, "%s shader has multiple %sputs sharing the " "same location that don't have the same " "interpolation qualification. Location %u " "component %u.\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", location, comp); return false; } if (info->centroid != centroid || info->sample != sample || info->patch != patch) { linker_error(prog, "%s shader has multiple %sputs sharing the " "same location that don't have the same " "auxiliary storage qualification. Location %u " "component %u.\n", _mesa_shader_stage_to_string(stage), var->data.mode == nir_var_shader_in ? "in" : "out", location, comp); return false; } } } else if (comp >= component && comp < last_comp) { info->var = var; info->base_type_is_integer = base_type_is_integer; info->base_type_bit_size = base_type_bit_size; info->interpolation = interpolation; info->centroid = centroid; info->sample = sample; info->patch = patch; } comp++; /* We need to do some special handling for doubles as dvec3 and * dvec4 consume two consecutive locations. We don't need to * worry about components beginning at anything other than 0 as * the spec does not allow this for dvec3 and dvec4. */ if (comp == 4 && last_comp > 4) { last_comp = last_comp - 4; /* Bump location index and reset the component index */ location++; comp = 0; component = 0; } } location++; } return true; } static void resize_input_array(nir_shader *shader, struct gl_shader_program *prog, unsigned stage, unsigned num_vertices) { nir_foreach_shader_in_variable(var, shader) { if (!glsl_type_is_array(var->type) || var->data.patch) continue; unsigned size = glsl_array_size(var->type); if (stage == MESA_SHADER_GEOMETRY) { /* Generate a link error if the shader has declared this array with * an incorrect size. */ if (!var->data.implicit_sized_array && size != -1 && size != num_vertices) { linker_error(prog, "size of array %s declared as %u, " "but number of input vertices is %u\n", var->name, size, num_vertices); break; } /* Generate a link error if the shader attempts to access an input * array using an index too large for its actual size assigned at * link time. */ if (var->data.max_array_access >= (int)num_vertices) { linker_error(prog, "%s shader accesses element %i of " "%s, but only %i input vertices\n", _mesa_shader_stage_to_string(stage), var->data.max_array_access, var->name, num_vertices); break; } } var->type = glsl_array_type(var->type->fields.array, num_vertices, 0); var->data.max_array_access = num_vertices - 1; } nir_fixup_deref_types(shader); } /** * Resize tessellation evaluation per-vertex inputs to the size of * tessellation control per-vertex outputs. */ void resize_tes_inputs(const struct gl_constants *consts, struct gl_shader_program *prog) { if (prog->_LinkedShaders[MESA_SHADER_TESS_EVAL] == NULL) return; struct gl_linked_shader *tcs = prog->_LinkedShaders[MESA_SHADER_TESS_CTRL]; struct gl_linked_shader *tes = prog->_LinkedShaders[MESA_SHADER_TESS_EVAL]; /* If no control shader is present, then the TES inputs are statically * sized to MaxPatchVertices; the actual size of the arrays won't be * known until draw time. */ const int num_vertices = tcs ? tcs->Program->info.tess.tcs_vertices_out : consts->MaxPatchVertices; resize_input_array(tes->Program->nir, prog, MESA_SHADER_TESS_EVAL, num_vertices); if (tcs) { /* Convert the gl_PatchVerticesIn system value into a constant, since * the value is known at this point. */ nir_variable *var = nir_find_variable_with_location(tes->Program->nir, nir_var_system_value, SYSTEM_VALUE_VERTICES_IN); if (var) { var->data.location = 0; var->data.explicit_location = false; var->data.mode = nir_var_mem_constant; nir_constant *val = rzalloc(var, nir_constant); val->values[0].i32 = num_vertices; var->constant_initializer = val; nir_fixup_deref_modes(tes->Program->nir); } } } void set_geom_shader_input_array_size(struct gl_shader_program *prog) { if (prog->_LinkedShaders[MESA_SHADER_GEOMETRY] == NULL) return; /* Set the size of geometry shader input arrays */ nir_shader *nir = prog->_LinkedShaders[MESA_SHADER_GEOMETRY]->Program->nir; unsigned num_vertices = mesa_vertices_per_prim(nir->info.gs.input_primitive); resize_input_array(nir, prog, MESA_SHADER_GEOMETRY, num_vertices); } static bool validate_explicit_variable_location(const struct gl_constants *consts, struct explicit_location_info explicit_locations[][4], nir_variable *var, struct gl_shader_program *prog, struct gl_linked_shader *sh) { const struct glsl_type *type = get_varying_type(var, sh->Stage); unsigned num_elements = glsl_count_attribute_slots(type, false); unsigned idx = compute_variable_location_slot(var, sh->Stage); unsigned slot_limit = idx + num_elements; /* Vertex shader inputs and fragment shader outputs are validated in * assign_attribute_or_color_locations() so we should not attempt to * validate them again here. */ unsigned slot_max; if (var->data.mode == nir_var_shader_out) { assert(sh->Stage != MESA_SHADER_FRAGMENT); slot_max = consts->Program[sh->Stage].MaxOutputComponents / 4; } else { assert(var->data.mode == nir_var_shader_in); assert(sh->Stage != MESA_SHADER_VERTEX); slot_max = consts->Program[sh->Stage].MaxInputComponents / 4; } if (slot_limit > slot_max) { linker_error(prog, "Invalid location %u in %s shader\n", idx, _mesa_shader_stage_to_string(sh->Stage)); return false; } const struct glsl_type *type_without_array = glsl_without_array(type); if (glsl_type_is_interface(type_without_array)) { for (unsigned i = 0; i < glsl_get_length(type_without_array); i++) { const struct glsl_struct_field *field = glsl_get_struct_field_data(type_without_array, i); unsigned field_location = field->location - (field->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0); unsigned field_slots = glsl_count_attribute_slots(field->type, false); if (!check_location_aliasing(explicit_locations, var, field_location, 0, field_location + field_slots, field->type, field->interpolation, field->centroid, field->sample, field->patch, prog, sh->Stage)) { return false; } } } else if (!check_location_aliasing(explicit_locations, var, idx, var->data.location_frac, slot_limit, type, var->data.interpolation, var->data.centroid, var->data.sample, var->data.patch, prog, sh->Stage)) { return false; } return true; } /** * Validate explicit locations for the inputs to the first stage and the * outputs of the last stage in a program, if those are not the VS and FS * shaders. */ void gl_nir_validate_first_and_last_interface_explicit_locations(const struct gl_constants *consts, struct gl_shader_program *prog, gl_shader_stage first_stage, gl_shader_stage last_stage) { /* VS inputs and FS outputs are validated in * assign_attribute_or_color_locations() */ bool validate_first_stage = first_stage != MESA_SHADER_VERTEX; bool validate_last_stage = last_stage != MESA_SHADER_FRAGMENT; if (!validate_first_stage && !validate_last_stage) return; struct explicit_location_info explicit_locations[MAX_VARYING][4]; gl_shader_stage stages[2] = { first_stage, last_stage }; bool validate_stage[2] = { validate_first_stage, validate_last_stage }; nir_variable_mode var_mode[2] = { nir_var_shader_in, nir_var_shader_out }; for (unsigned i = 0; i < 2; i++) { if (!validate_stage[i]) continue; gl_shader_stage stage = stages[i]; struct gl_linked_shader *sh = prog->_LinkedShaders[stage]; assert(sh); memset(explicit_locations, 0, sizeof(explicit_locations)); nir_foreach_variable_with_modes(var, sh->Program->nir, var_mode[i]) { if (!var->data.explicit_location || var->data.location < VARYING_SLOT_VAR0) continue; if (!validate_explicit_variable_location(consts, explicit_locations, var, prog, sh)) { return; } } } } /** * Check if we should force input / output matching between shader * interfaces. * * Section 4.3.4 (Inputs) of the GLSL 4.10 specifications say: * * "Only the input variables that are actually read need to be * written by the previous stage; it is allowed to have * superfluous declarations of input variables." * * However it's not defined anywhere as to how we should handle * inputs that are not written in the previous stage and it's not * clear what "actually read" means. * * The GLSL 4.20 spec however is much clearer: * * "Only the input variables that are statically read need to * be written by the previous stage; it is allowed to have * superfluous declarations of input variables." * * It also has a table that states it is an error to statically * read an input that is not defined in the previous stage. While * it is not an error to not statically write to the output (it * just needs to be defined to not be an error). * * The text in the GLSL 4.20 spec was an attempt to clarify the * previous spec iterations. However given the difference in spec * and that some applications seem to depend on not erroring when * the input is not actually read in control flow we only apply * this rule to GLSL 4.20 and higher. GLSL 4.10 shaders have been * seen in the wild that depend on the less strict interpretation. */ static bool static_input_output_matching(struct gl_shader_program *prog) { return prog->GLSL_Version >= (prog->IsES ? 0 : 420); } /** * Validate that outputs from one stage match inputs of another */ void gl_nir_cross_validate_outputs_to_inputs(const struct gl_constants *consts, struct gl_shader_program *prog, struct gl_linked_shader *producer, struct gl_linked_shader *consumer) { struct _mesa_symbol_table *table = _mesa_symbol_table_ctor(); struct explicit_location_info output_explicit_locations[MAX_VARYING][4] = {0}; struct explicit_location_info input_explicit_locations[MAX_VARYING][4] = {0}; /* Find all shader outputs in the "producer" stage. */ nir_foreach_variable_with_modes(var, producer->Program->nir, nir_var_shader_out) { if (!var->data.explicit_location || var->data.location < VARYING_SLOT_VAR0) { /* Interface block validation is handled elsewhere */ if (!var->interface_type || is_gl_identifier(var->name)) _mesa_symbol_table_add_symbol(table, var->name, var); } else { /* User-defined varyings with explicit locations are handled * differently because they do not need to have matching names. */ if (!validate_explicit_variable_location(consts, output_explicit_locations, var, prog, producer)) { goto out; } } } /* Find all shader inputs in the "consumer" stage. Any variables that have * matching outputs already in the symbol table must have the same type and * qualifiers. * * Exception: if the consumer is the geometry shader, then the inputs * should be arrays and the type of the array element should match the type * of the corresponding producer output. */ nir_foreach_variable_with_modes(input, consumer->Program->nir, nir_var_shader_in) { if (strcmp(input->name, "gl_Color") == 0 && input->data.used) { const nir_variable *front_color = (nir_variable *) _mesa_symbol_table_find_symbol(table, "gl_FrontColor"); const nir_variable *back_color = (nir_variable *) _mesa_symbol_table_find_symbol(table, "gl_BackColor"); cross_validate_front_and_back_color(consts, prog, input, front_color, back_color, consumer->Stage, producer->Stage); } else if (strcmp(input->name, "gl_SecondaryColor") == 0 && input->data.used) { const nir_variable *front_color = (nir_variable *) _mesa_symbol_table_find_symbol(table, "gl_FrontSecondaryColor"); const nir_variable *back_color = (nir_variable *) _mesa_symbol_table_find_symbol(table, "gl_BackSecondaryColor"); cross_validate_front_and_back_color(consts, prog, input, front_color, back_color, consumer->Stage, producer->Stage); } else { /* The rules for connecting inputs and outputs change in the presence * of explicit locations. In this case, we no longer care about the * names of the variables. Instead, we care only about the * explicitly assigned location. */ nir_variable *output = NULL; if (input->data.explicit_location && input->data.location >= VARYING_SLOT_VAR0) { const struct glsl_type *type = get_varying_type(input, consumer->Stage); unsigned num_elements = glsl_count_attribute_slots(type, false); unsigned idx = compute_variable_location_slot(input, consumer->Stage); unsigned slot_limit = idx + num_elements; if (!validate_explicit_variable_location(consts, input_explicit_locations, input, prog, consumer)) { goto out; } while (idx < slot_limit) { if (idx >= MAX_VARYING) { linker_error(prog, "Invalid location %u in %s shader\n", idx, _mesa_shader_stage_to_string(consumer->Stage)); goto out; } output = output_explicit_locations[idx][input->data.location_frac].var; if (output == NULL) { /* A linker failure should only happen when there is no * output declaration and there is Static Use of the * declared input. */ if (input->data.used && static_input_output_matching(prog)) { linker_error(prog, "%s shader input `%s' with explicit location " "has no matching output\n", _mesa_shader_stage_to_string(consumer->Stage), input->name); break; } } else if (input->data.location != output->data.location) { linker_error(prog, "%s shader input `%s' with explicit location " "has no matching output\n", _mesa_shader_stage_to_string(consumer->Stage), input->name); break; } idx++; } } else { /* Interface block validation is handled elsewhere */ if (input->interface_type) continue; output = (nir_variable *) _mesa_symbol_table_find_symbol(table, input->name); } if (output != NULL) { /* Interface blocks have their own validation elsewhere so don't * try validating them here. */ if (!(input->interface_type && output->interface_type)) cross_validate_types_and_qualifiers(consts, prog, input, output, consumer->Stage, producer->Stage); } else { /* Check for input vars with unmatched output vars in prev stage * taking into account that interface blocks could have a matching * output but with different name, so we ignore them. */ assert(!input->data.assigned); if (input->data.used && !input->interface_type && !input->data.explicit_location && static_input_output_matching(prog)) linker_error(prog, "%s shader input `%s' " "has no matching output in the previous stage\n", _mesa_shader_stage_to_string(consumer->Stage), input->name); } } } out: _mesa_symbol_table_dtor(table); } /** * Assign locations for either VS inputs or FS outputs. * * \param mem_ctx Temporary ralloc context used for linking. * \param prog Shader program whose variables need locations * assigned. * \param constants Driver specific constant values for the program. * \param target_index Selector for the program target to receive location * assignmnets. Must be either \c MESA_SHADER_VERTEX or * \c MESA_SHADER_FRAGMENT. * \param do_assignment Whether we are actually marking the assignment or we * are just doing a dry-run checking. * * \return * If locations are (or can be, in case of dry-running) successfully assigned, * true is returned. Otherwise an error is emitted to the shader link log and * false is returned. */ static bool assign_attribute_or_color_locations(void *mem_ctx, struct gl_shader_program *prog, const struct gl_constants *constants, unsigned target_index, bool do_assignment) { /* Maximum number of generic locations. This corresponds to either the * maximum number of draw buffers or the maximum number of generic * attributes. */ unsigned max_index = (target_index == MESA_SHADER_VERTEX) ? constants->Program[target_index].MaxAttribs : MAX2(constants->MaxDrawBuffers, constants->MaxDualSourceDrawBuffers); assert(max_index <= 32); struct temp_attr to_assign[32]; /* Mark invalid locations as being used. */ unsigned used_locations = ~SAFE_MASK_FROM_INDEX(max_index); unsigned double_storage_locations = 0; assert((target_index == MESA_SHADER_VERTEX) || (target_index == MESA_SHADER_FRAGMENT)); if (prog->_LinkedShaders[target_index] == NULL) return true; /* Operate in a total of four passes. * * 1. Invalidate the location assignments for all vertex shader inputs. * * 2. Assign locations for inputs that have user-defined (via * glBindVertexAttribLocation) locations and outputs that have * user-defined locations (via glBindFragDataLocation). * * 3. Sort the attributes without assigned locations by number of slots * required in decreasing order. Fragmentation caused by attribute * locations assigned by the application may prevent large attributes * from having enough contiguous space. * * 4. Assign locations to any inputs without assigned locations. */ const int generic_base = (target_index == MESA_SHADER_VERTEX) ? (int) VERT_ATTRIB_GENERIC0 : (int) FRAG_RESULT_DATA0; nir_variable_mode io_mode = (target_index == MESA_SHADER_VERTEX) ? nir_var_shader_in : nir_var_shader_out; /* Temporary array for the set of attributes that have locations assigned, * for the purpose of checking overlapping slots/components of (non-ES) * fragment shader outputs. */ nir_variable *assigned[FRAG_RESULT_MAX * 4]; /* (max # of FS outputs) * # components */ unsigned assigned_attr = 0; unsigned num_attr = 0; nir_shader *shader = prog->_LinkedShaders[target_index]->Program->nir; nir_foreach_variable_with_modes(var, shader, io_mode) { if (var->data.explicit_location) { if ((var->data.location >= (int)(max_index + generic_base)) || (var->data.location < 0)) { linker_error(prog, "invalid explicit location %d specified for `%s'\n", (var->data.location < 0) ? var->data.location : var->data.location - generic_base, var->name); return false; } } else if (target_index == MESA_SHADER_VERTEX) { unsigned binding; if (string_to_uint_map_get(prog->AttributeBindings, &binding, var->name)) { assert(binding >= VERT_ATTRIB_GENERIC0); var->data.location = binding; } } else if (target_index == MESA_SHADER_FRAGMENT) { unsigned binding; unsigned index; const char *name = var->name; const struct glsl_type *type = var->type; while (type) { /* Check if there's a binding for the variable name */ if (string_to_uint_map_get(prog->FragDataBindings, &binding, name)) { assert(binding >= FRAG_RESULT_DATA0); var->data.location = binding; if (string_to_uint_map_get(prog->FragDataIndexBindings, &index, name)) { var->data.index = index; } break; } /* If not, but it's an array type, look for name[0] */ if (glsl_type_is_array(type)) { name = ralloc_asprintf(mem_ctx, "%s[0]", name); type = glsl_get_array_element(type); continue; } break; } } if (strcmp(var->name, "gl_LastFragData") == 0) continue; /* From GL4.5 core spec, section 15.2 (Shader Execution): * * "Output binding assignments will cause LinkProgram to fail: * ... * If the program has an active output assigned to a location greater * than or equal to the value of MAX_DUAL_SOURCE_DRAW_BUFFERS and has * an active output assigned an index greater than or equal to one;" */ if (target_index == MESA_SHADER_FRAGMENT && var->data.index >= 1 && var->data.location - generic_base >= (int) constants->MaxDualSourceDrawBuffers) { linker_error(prog, "output location %d >= GL_MAX_DUAL_SOURCE_DRAW_BUFFERS " "with index %u for %s\n", var->data.location - generic_base, var->data.index, var->name); return false; } const unsigned slots = glsl_count_attribute_slots(var->type, target_index == MESA_SHADER_VERTEX); /* If the variable is not a built-in and has a location statically * assigned in the shader (presumably via a layout qualifier), make sure * that it doesn't collide with other assigned locations. Otherwise, * add it to the list of variables that need linker-assigned locations. */ if (var->data.location != -1) { if (var->data.location >= generic_base && var->data.index < 1) { /* From page 61 of the OpenGL 4.0 spec: * * "LinkProgram will fail if the attribute bindings assigned * by BindAttribLocation do not leave not enough space to * assign a location for an active matrix attribute or an * active attribute array, both of which require multiple * contiguous generic attributes." * * I think above text prohibits the aliasing of explicit and * automatic assignments. But, aliasing is allowed in manual * assignments of attribute locations. See below comments for * the details. * * From OpenGL 4.0 spec, page 61: * * "It is possible for an application to bind more than one * attribute name to the same location. This is referred to as * aliasing. This will only work if only one of the aliased * attributes is active in the executable program, or if no * path through the shader consumes more than one attribute of * a set of attributes aliased to the same location. A link * error can occur if the linker determines that every path * through the shader consumes multiple aliased attributes, * but implementations are not required to generate an error * in this case." * * From GLSL 4.30 spec, page 54: * * "A program will fail to link if any two non-vertex shader * input variables are assigned to the same location. For * vertex shaders, multiple input variables may be assigned * to the same location using either layout qualifiers or via * the OpenGL API. However, such aliasing is intended only to * support vertex shaders where each execution path accesses * at most one input per each location. Implementations are * permitted, but not required, to generate link-time errors * if they detect that every path through the vertex shader * executable accesses multiple inputs assigned to any single * location. For all shader types, a program will fail to link * if explicit location assignments leave the linker unable * to find space for other variables without explicit * assignments." * * From OpenGL ES 3.0 spec, page 56: * * "Binding more than one attribute name to the same location * is referred to as aliasing, and is not permitted in OpenGL * ES Shading Language 3.00 vertex shaders. LinkProgram will * fail when this condition exists. However, aliasing is * possible in OpenGL ES Shading Language 1.00 vertex shaders. * This will only work if only one of the aliased attributes * is active in the executable program, or if no path through * the shader consumes more than one attribute of a set of * attributes aliased to the same location. A link error can * occur if the linker determines that every path through the * shader consumes multiple aliased attributes, but implemen- * tations are not required to generate an error in this case." * * After looking at above references from OpenGL, OpenGL ES and * GLSL specifications, we allow aliasing of vertex input variables * in: OpenGL 2.0 (and above) and OpenGL ES 2.0. * * NOTE: This is not required by the spec but its worth mentioning * here that we're not doing anything to make sure that no path * through the vertex shader executable accesses multiple inputs * assigned to any single location. */ /* Mask representing the contiguous slots that will be used by * this attribute. */ const unsigned attr = var->data.location - generic_base; const unsigned use_mask = (1 << slots) - 1; const char *const string = (target_index == MESA_SHADER_VERTEX) ? "vertex shader input" : "fragment shader output"; /* Generate a link error if the requested locations for this * attribute exceed the maximum allowed attribute location. */ if (attr + slots > max_index) { linker_error(prog, "insufficient contiguous locations " "available for %s `%s' %d %d %d\n", string, var->name, used_locations, use_mask, attr); return false; } /* Generate a link error if the set of bits requested for this * attribute overlaps any previously allocated bits. */ if ((~(use_mask << attr) & used_locations) != used_locations) { if (target_index == MESA_SHADER_FRAGMENT && !prog->IsES) { /* From section 4.4.2 (Output Layout Qualifiers) of the GLSL * 4.40 spec: * * "Additionally, for fragment shader outputs, if two * variables are placed within the same location, they * must have the same underlying type (floating-point or * integer). No component aliasing of output variables or * members is allowed. */ for (unsigned i = 0; i < assigned_attr; i++) { unsigned assigned_slots = glsl_count_attribute_slots(assigned[i]->type, false); unsigned assig_attr = assigned[i]->data.location - generic_base; unsigned assigned_use_mask = (1 << assigned_slots) - 1; if ((assigned_use_mask << assig_attr) & (use_mask << attr)) { const struct glsl_type *assigned_type = glsl_without_array(assigned[i]->type); const struct glsl_type *type = glsl_without_array(var->type); if (glsl_get_base_type(assigned_type) != glsl_get_base_type(type)) { linker_error(prog, "types do not match for aliased" " %ss %s and %s\n", string, assigned[i]->name, var->name); return false; } unsigned assigned_component_mask = ((1 << glsl_get_vector_elements(assigned_type)) - 1) << assigned[i]->data.location_frac; unsigned component_mask = ((1 << glsl_get_vector_elements(type)) - 1) << var->data.location_frac; if (assigned_component_mask & component_mask) { linker_error(prog, "overlapping component is " "assigned to %ss %s and %s " "(component=%d)\n", string, assigned[i]->name, var->name, var->data.location_frac); return false; } } } } else if (target_index == MESA_SHADER_FRAGMENT || (prog->IsES && prog->GLSL_Version >= 300)) { linker_error(prog, "overlapping location is assigned " "to %s `%s' %d %d %d\n", string, var->name, used_locations, use_mask, attr); return false; } else { linker_warning(prog, "overlapping location is assigned " "to %s `%s' %d %d %d\n", string, var->name, used_locations, use_mask, attr); } } if (target_index == MESA_SHADER_FRAGMENT && !prog->IsES) { /* Only track assigned variables for non-ES fragment shaders * to avoid overflowing the array. * * At most one variable per fragment output component should * reach this. */ assert(assigned_attr < ARRAY_SIZE(assigned)); assigned[assigned_attr] = var; assigned_attr++; } used_locations |= (use_mask << attr); /* From the GL 4.5 core spec, section 11.1.1 (Vertex Attributes): * * "A program with more than the value of MAX_VERTEX_ATTRIBS * active attribute variables may fail to link, unless * device-dependent optimizations are able to make the program * fit within available hardware resources. For the purposes * of this test, attribute variables of the type dvec3, dvec4, * dmat2x3, dmat2x4, dmat3, dmat3x4, dmat4x3, and dmat4 may * count as consuming twice as many attributes as equivalent * single-precision types. While these types use the same number * of generic attributes as their single-precision equivalents, * implementations are permitted to consume two single-precision * vectors of internal storage for each three- or four-component * double-precision vector." * * Mark this attribute slot as taking up twice as much space * so we can count it properly against limits. According to * issue (3) of the GL_ARB_vertex_attrib_64bit behavior, this * is optional behavior, but it seems preferable. */ if (glsl_type_is_dual_slot(glsl_without_array(var->type))) double_storage_locations |= (use_mask << attr); } continue; } if (num_attr >= max_index) { linker_error(prog, "too many %s (max %u)", target_index == MESA_SHADER_VERTEX ? "vertex shader inputs" : "fragment shader outputs", max_index); return false; } to_assign[num_attr].slots = slots; to_assign[num_attr].var = var; to_assign[num_attr].original_idx = num_attr; num_attr++; } if (!do_assignment) return true; if (target_index == MESA_SHADER_VERTEX) { unsigned total_attribs_size = util_bitcount(used_locations & SAFE_MASK_FROM_INDEX(max_index)) + util_bitcount(double_storage_locations); if (total_attribs_size > max_index) { linker_error(prog, "attempt to use %d vertex attribute slots only %d available ", total_attribs_size, max_index); return false; } } /* If all of the attributes were assigned locations by the application (or * are built-in attributes with fixed locations), return early. This should * be the common case. */ if (num_attr == 0) return true; qsort(to_assign, num_attr, sizeof(to_assign[0]), &compare_attr); if (target_index == MESA_SHADER_VERTEX) { /* VERT_ATTRIB_GENERIC0 is a pseudo-alias for VERT_ATTRIB_POS. It can * only be explicitly assigned by via glBindAttribLocation. Mark it as * reserved to prevent it from being automatically allocated below. */ if (find_deref(shader, "gl_Vertex")) used_locations |= (1 << 0); } for (unsigned i = 0; i < num_attr; i++) { /* Mask representing the contiguous slots that will be used by this * attribute. */ const unsigned use_mask = (1 << to_assign[i].slots) - 1; int location = find_available_slots(used_locations, to_assign[i].slots); if (location < 0) { const char *const string = (target_index == MESA_SHADER_VERTEX) ? "vertex shader input" : "fragment shader output"; linker_error(prog, "insufficient contiguous locations " "available for %s `%s'\n", string, to_assign[i].var->name); return false; } to_assign[i].var->data.location = generic_base + location; used_locations |= (use_mask << location); if (glsl_type_is_dual_slot(glsl_without_array(to_assign[i].var->type))) double_storage_locations |= (use_mask << location); } /* Now that we have all the locations, from the GL 4.5 core spec, section * 11.1.1 (Vertex Attributes), dvec3, dvec4, dmat2x3, dmat2x4, dmat3, * dmat3x4, dmat4x3, and dmat4 count as consuming twice as many attributes * as equivalent single-precision types. */ if (target_index == MESA_SHADER_VERTEX) { unsigned total_attribs_size = util_bitcount(used_locations & SAFE_MASK_FROM_INDEX(max_index)) + util_bitcount(double_storage_locations); if (total_attribs_size > max_index) { linker_error(prog, "attempt to use %d vertex attribute slots only %d available ", total_attribs_size, max_index); return false; } } return true; } static bool varying_has_user_specified_location(const nir_variable *var) { return var->data.explicit_location && var->data.location >= VARYING_SLOT_VAR0; } static void create_xfb_varying_names(void *mem_ctx, const struct glsl_type *t, char **name, size_t name_length, unsigned *count, const char *ifc_member_name, const struct glsl_type *ifc_member_t, char ***varying_names) { if (glsl_type_is_interface(t)) { size_t new_length = name_length; assert(ifc_member_name && ifc_member_t); ralloc_asprintf_rewrite_tail(name, &new_length, ".%s", ifc_member_name); create_xfb_varying_names(mem_ctx, ifc_member_t, name, new_length, count, NULL, NULL, varying_names); } else if (glsl_type_is_struct(t)) { for (unsigned i = 0; i < glsl_get_length(t); i++) { const char *field = glsl_get_struct_elem_name(t, i); size_t new_length = name_length; ralloc_asprintf_rewrite_tail(name, &new_length, ".%s", field); create_xfb_varying_names(mem_ctx, glsl_get_struct_field(t, i), name, new_length, count, NULL, NULL, varying_names); } } else if (glsl_type_is_struct(glsl_without_array(t)) || glsl_type_is_interface(glsl_without_array(t)) || (glsl_type_is_array(t) && glsl_type_is_array(glsl_get_array_element(t)))) { for (unsigned i = 0; i < glsl_get_length(t); i++) { size_t new_length = name_length; /* Append the subscript to the current variable name */ ralloc_asprintf_rewrite_tail(name, &new_length, "[%u]", i); create_xfb_varying_names(mem_ctx, glsl_get_array_element(t), name, new_length, count, ifc_member_name, ifc_member_t, varying_names); } } else { (*varying_names)[(*count)++] = ralloc_strdup(mem_ctx, *name); } } static bool process_xfb_layout_qualifiers(void *mem_ctx, const struct gl_linked_shader *sh, struct gl_shader_program *prog, unsigned *num_xfb_decls, char ***varying_names, bool *compact_arrays) { bool has_xfb_qualifiers = false; /* We still need to enable transform feedback mode even if xfb_stride is * only applied to a global out. Also we don't bother to propagate * xfb_stride to interface block members so this will catch that case also. */ for (unsigned j = 0; j < MAX_FEEDBACK_BUFFERS; j++) { if (prog->TransformFeedback.BufferStride[j]) { has_xfb_qualifiers = true; break; } } *compact_arrays = sh->Program->nir->options->compact_arrays; nir_foreach_shader_out_variable(var, sh->Program->nir) { /* From the ARB_enhanced_layouts spec: * * "Any shader making any static use (after preprocessing) of any of * these *xfb_* qualifiers will cause the shader to be in a * transform feedback capturing mode and hence responsible for * describing the transform feedback setup. This mode will capture * any output selected by *xfb_offset*, directly or indirectly, to * a transform feedback buffer." */ if (var->data.explicit_xfb_buffer || var->data.explicit_xfb_stride) { has_xfb_qualifiers = true; } if (var->data.explicit_offset) { *num_xfb_decls += glsl_varying_count(var->type); has_xfb_qualifiers = true; } } if (*num_xfb_decls == 0) return has_xfb_qualifiers; unsigned i = 0; *varying_names = ralloc_array(mem_ctx, char *, *num_xfb_decls); nir_foreach_shader_out_variable(var, sh->Program->nir) { if (var->data.explicit_offset) { char *name; const struct glsl_type *type, *member_type; if (var->data.from_named_ifc_block) { type = var->interface_type; /* Find the member type before it was altered by lowering */ const struct glsl_type *type_wa = glsl_without_array(type); member_type = glsl_get_struct_field(type_wa, glsl_get_field_index(type_wa, var->name)); name = ralloc_strdup(NULL, glsl_get_type_name(type_wa)); } else { type = var->type; member_type = NULL; name = ralloc_strdup(NULL, var->name); } create_xfb_varying_names(mem_ctx, type, &name, strlen(name), &i, var->name, member_type, varying_names); ralloc_free(name); } } assert(i == *num_xfb_decls); return has_xfb_qualifiers; } /** * Initialize this struct based on a string that was passed to * glTransformFeedbackVaryings. * * If the input is mal-formed, this call still succeeds, but it sets * this->var_name to a mal-formed input, so xfb_decl_find_output_var() * will fail to find any matching variable. */ static void xfb_decl_init(struct xfb_decl *xfb_decl, const struct gl_constants *consts, const struct gl_extensions *exts, const void *mem_ctx, const char *input, bool compact_arrays) { /* We don't have to be pedantic about what is a valid GLSL variable name, * because any variable with an invalid name can't exist in the IR anyway. */ xfb_decl->location = -1; xfb_decl->orig_name = input; xfb_decl->lowered_builtin_array_variable = none; xfb_decl->skip_components = 0; xfb_decl->next_buffer_separator = false; xfb_decl->matched_candidate = NULL; xfb_decl->stream_id = 0; xfb_decl->buffer = 0; xfb_decl->offset = 0; if (exts->ARB_transform_feedback3) { /* Parse gl_NextBuffer. */ if (strcmp(input, "gl_NextBuffer") == 0) { xfb_decl->next_buffer_separator = true; return; } /* Parse gl_SkipComponents. */ if (strcmp(input, "gl_SkipComponents1") == 0) xfb_decl->skip_components = 1; else if (strcmp(input, "gl_SkipComponents2") == 0) xfb_decl->skip_components = 2; else if (strcmp(input, "gl_SkipComponents3") == 0) xfb_decl->skip_components = 3; else if (strcmp(input, "gl_SkipComponents4") == 0) xfb_decl->skip_components = 4; if (xfb_decl->skip_components) return; } /* Parse a declaration. */ const char *base_name_end; long subscript = link_util_parse_program_resource_name(input, strlen(input), &base_name_end); xfb_decl->var_name = ralloc_strndup(mem_ctx, input, base_name_end - input); if (xfb_decl->var_name == NULL) { _mesa_error_no_memory(__func__); return; } if (subscript >= 0) { xfb_decl->array_subscript = subscript; xfb_decl->is_subscripted = true; } else { xfb_decl->is_subscripted = false; } /* For drivers that lower gl_ClipDistance to gl_ClipDistanceMESA, this * class must behave specially to account for the fact that gl_ClipDistance * is converted from a float[8] to a vec4[2]. */ if (!compact_arrays && strcmp(xfb_decl->var_name, "gl_ClipDistance") == 0) { xfb_decl->lowered_builtin_array_variable = clip_distance; } if (!compact_arrays && strcmp(xfb_decl->var_name, "gl_CullDistance") == 0) { xfb_decl->lowered_builtin_array_variable = cull_distance; } } /** * Determine whether two xfb_decl structs refer to the same variable and * array index (if applicable). */ static bool xfb_decl_is_same(const struct xfb_decl *x, const struct xfb_decl *y) { assert(xfb_decl_is_varying(x) && xfb_decl_is_varying(y)); if (strcmp(x->var_name, y->var_name) != 0) return false; if (x->is_subscripted != y->is_subscripted) return false; if (x->is_subscripted && x->array_subscript != y->array_subscript) return false; return true; } /** * The total number of varying components taken up by this variable. Only * valid if assign_location() has been called. */ static unsigned xfb_decl_num_components(struct xfb_decl *xfb_decl) { if (xfb_decl->lowered_builtin_array_variable) return xfb_decl->size; else return xfb_decl->vector_elements * xfb_decl->matrix_columns * xfb_decl->size * (_mesa_gl_datatype_is_64bit(xfb_decl->type) ? 2 : 1); } /** * Assign a location and stream ID for this xfb_decl object based on the * transform feedback candidate found by find_candidate. * * If an error occurs, the error is reported through linker_error() and false * is returned. */ static bool xfb_decl_assign_location(struct xfb_decl *xfb_decl, const struct gl_constants *consts, struct gl_shader_program *prog, bool disable_varying_packing, bool xfb_enabled) { assert(xfb_decl_is_varying(xfb_decl)); unsigned fine_location = xfb_decl->matched_candidate->toplevel_var->data.location * 4 + xfb_decl->matched_candidate->toplevel_var->data.location_frac + xfb_decl->matched_candidate->struct_offset_floats; const unsigned dmul = glsl_type_is_64bit(glsl_without_array(xfb_decl->matched_candidate->type)) ? 2 : 1; if (glsl_type_is_array(xfb_decl->matched_candidate->type)) { /* Array variable */ const struct glsl_type *element_type = glsl_get_array_element(xfb_decl->matched_candidate->type); const unsigned matrix_cols = glsl_get_matrix_columns(element_type); const unsigned vector_elements = glsl_get_vector_elements(element_type); unsigned actual_array_size; switch (xfb_decl->lowered_builtin_array_variable) { case clip_distance: actual_array_size = prog->last_vert_prog ? prog->last_vert_prog->nir->info.clip_distance_array_size : 0; break; case cull_distance: actual_array_size = prog->last_vert_prog ? prog->last_vert_prog->nir->info.cull_distance_array_size : 0; break; case none: default: actual_array_size = glsl_array_size(xfb_decl->matched_candidate->type); break; } if (xfb_decl->is_subscripted) { /* Check array bounds. */ if (xfb_decl->array_subscript >= actual_array_size) { linker_error(prog, "Transform feedback varying %s has index " "%i, but the array size is %u.", xfb_decl->orig_name, xfb_decl->array_subscript, actual_array_size); return false; } bool array_will_be_lowered = lower_packed_varying_needs_lowering(prog->last_vert_prog->nir, xfb_decl->matched_candidate->toplevel_var, nir_var_shader_out, disable_varying_packing, xfb_enabled) || strcmp(xfb_decl->matched_candidate->toplevel_var->name, "gl_ClipDistance") == 0 || strcmp(xfb_decl->matched_candidate->toplevel_var->name, "gl_CullDistance") == 0 || strcmp(xfb_decl->matched_candidate->toplevel_var->name, "gl_TessLevelInner") == 0 || strcmp(xfb_decl->matched_candidate->toplevel_var->name, "gl_TessLevelOuter") == 0; unsigned array_elem_size = xfb_decl->lowered_builtin_array_variable ? 1 : (array_will_be_lowered ? vector_elements : 4) * matrix_cols * dmul; fine_location += array_elem_size * xfb_decl->array_subscript; xfb_decl->size = 1; } else { xfb_decl->size = actual_array_size; } xfb_decl->vector_elements = vector_elements; xfb_decl->matrix_columns = matrix_cols; if (xfb_decl->lowered_builtin_array_variable) xfb_decl->type = GL_FLOAT; else xfb_decl->type = glsl_get_gl_type(element_type); } else { /* Regular variable (scalar, vector, or matrix) */ if (xfb_decl->is_subscripted) { linker_error(prog, "Transform feedback varying %s requested, " "but %s is not an array.", xfb_decl->orig_name, xfb_decl->var_name); return false; } xfb_decl->size = 1; xfb_decl->vector_elements = glsl_get_vector_elements(xfb_decl->matched_candidate->type); xfb_decl->matrix_columns = glsl_get_matrix_columns(xfb_decl->matched_candidate->type); xfb_decl->type = glsl_get_gl_type(xfb_decl->matched_candidate->type); } xfb_decl->location = fine_location / 4; xfb_decl->location_frac = fine_location % 4; /* From GL_EXT_transform_feedback: * A program will fail to link if: * * * the total number of components to capture in any varying * variable in is greater than the constant * MAX_TRANSFORM_FEEDBACK_SEPARATE_COMPONENTS_EXT and the * buffer mode is SEPARATE_ATTRIBS_EXT; */ if (prog->TransformFeedback.BufferMode == GL_SEPARATE_ATTRIBS && xfb_decl_num_components(xfb_decl) > consts->MaxTransformFeedbackSeparateComponents) { linker_error(prog, "Transform feedback varying %s exceeds " "MAX_TRANSFORM_FEEDBACK_SEPARATE_COMPONENTS.", xfb_decl->orig_name); return false; } /* Only transform feedback varyings can be assigned to non-zero streams, * so assign the stream id here. */ xfb_decl->stream_id = xfb_decl->matched_candidate->toplevel_var->data.stream; unsigned array_offset = xfb_decl->array_subscript * 4 * dmul; unsigned struct_offset = xfb_decl->matched_candidate->xfb_offset_floats * 4; xfb_decl->buffer = xfb_decl->matched_candidate->toplevel_var->data.xfb.buffer; xfb_decl->offset = xfb_decl->matched_candidate->toplevel_var->data.offset + array_offset + struct_offset; return true; } static unsigned xfb_decl_get_num_outputs(struct xfb_decl *xfb_decl) { if (!xfb_decl_is_varying(xfb_decl)) { return 0; } if (varying_has_user_specified_location(xfb_decl->matched_candidate->toplevel_var)) { unsigned dmul = _mesa_gl_datatype_is_64bit(xfb_decl->type) ? 2 : 1; unsigned rows_per_element = DIV_ROUND_UP(xfb_decl->vector_elements * dmul, 4); return xfb_decl->size * xfb_decl->matrix_columns * rows_per_element; } else { return (xfb_decl_num_components(xfb_decl) + xfb_decl->location_frac + 3) / 4; } } static bool xfb_decl_is_varying_written(struct xfb_decl *xfb_decl) { if (xfb_decl->next_buffer_separator || xfb_decl->skip_components) return false; return xfb_decl->matched_candidate->toplevel_var->data.assigned; } /** * Update gl_transform_feedback_info to reflect this xfb_decl. * * If an error occurs, the error is reported through linker_error() and false * is returned. */ static bool xfb_decl_store(struct xfb_decl *xfb_decl, const struct gl_constants *consts, struct gl_shader_program *prog, struct gl_transform_feedback_info *info, unsigned buffer, unsigned buffer_index, const unsigned max_outputs, BITSET_WORD *used_components[MAX_FEEDBACK_BUFFERS], bool *explicit_stride, unsigned *max_member_alignment, bool has_xfb_qualifiers, const void* mem_ctx) { unsigned xfb_offset = 0; unsigned size = xfb_decl->size; /* Handle gl_SkipComponents. */ if (xfb_decl->skip_components) { info->Buffers[buffer].Stride += xfb_decl->skip_components; size = xfb_decl->skip_components; goto store_varying; } if (xfb_decl->next_buffer_separator) { size = 0; goto store_varying; } if (has_xfb_qualifiers) { xfb_offset = xfb_decl->offset / 4; } else { xfb_offset = info->Buffers[buffer].Stride; } info->Varyings[info->NumVarying].Offset = xfb_offset * 4; { unsigned location = xfb_decl->location; unsigned location_frac = xfb_decl->location_frac; unsigned num_components = xfb_decl_num_components(xfb_decl); /* From GL_EXT_transform_feedback: * * " A program will fail to link if: * * * the total number of components to capture is greater than the * constant MAX_TRANSFORM_FEEDBACK_INTERLEAVED_COMPONENTS_EXT * and the buffer mode is INTERLEAVED_ATTRIBS_EXT." * * From GL_ARB_enhanced_layouts: * * " The resulting stride (implicit or explicit) must be less than or * equal to the implementation-dependent constant * gl_MaxTransformFeedbackInterleavedComponents." */ if ((prog->TransformFeedback.BufferMode == GL_INTERLEAVED_ATTRIBS || has_xfb_qualifiers) && xfb_offset + num_components > consts->MaxTransformFeedbackInterleavedComponents) { linker_error(prog, "The MAX_TRANSFORM_FEEDBACK_INTERLEAVED_COMPONENTS " "limit has been exceeded."); return false; } /* From the OpenGL 4.60.5 spec, section 4.4.2. Output Layout Qualifiers, * Page 76, (Transform Feedback Layout Qualifiers): * * " No aliasing in output buffers is allowed: It is a compile-time or * link-time error to specify variables with overlapping transform * feedback offsets." */ const unsigned max_components = consts->MaxTransformFeedbackInterleavedComponents; const unsigned first_component = xfb_offset; const unsigned last_component = xfb_offset + num_components - 1; const unsigned start_word = BITSET_BITWORD(first_component); const unsigned end_word = BITSET_BITWORD(last_component); BITSET_WORD *used; assert(last_component < max_components); if (!used_components[buffer]) { used_components[buffer] = rzalloc_array(mem_ctx, BITSET_WORD, BITSET_WORDS(max_components)); } used = used_components[buffer]; for (unsigned word = start_word; word <= end_word; word++) { unsigned start_range = 0; unsigned end_range = BITSET_WORDBITS - 1; if (word == start_word) start_range = first_component % BITSET_WORDBITS; if (word == end_word) end_range = last_component % BITSET_WORDBITS; if (used[word] & BITSET_RANGE(start_range, end_range)) { linker_error(prog, "variable '%s', xfb_offset (%d) is causing aliasing.", xfb_decl->orig_name, xfb_offset * 4); return false; } used[word] |= BITSET_RANGE(start_range, end_range); } const unsigned type_num_components = xfb_decl->vector_elements * (_mesa_gl_datatype_is_64bit(xfb_decl->type) ? 2 : 1); unsigned current_type_components_left = type_num_components; while (num_components > 0) { unsigned output_size = 0; /* From GL_ARB_enhanced_layouts: * * "When an attribute variable declared using an array type is bound to * generic attribute index , the active array elements are assigned to * consecutive generic attributes beginning with generic attribute . The * number of attributes and components assigned to each element are * determined according to the data type of array elements and "component" * layout qualifier (if any) specified in the declaration of the array." * * "When an attribute variable declared using a matrix type is bound to a * generic attribute index , its values are taken from consecutive generic * attributes beginning with generic attribute . Such matrices are * treated as an array of column vectors with values taken from the generic * attributes. * This means there may be gaps in the varyings we are taking values from." * * Examples: * * | layout(location=0) dvec3[2] a; | layout(location=4) vec2[4] b; | * | | | * | 32b 32b 32b 32b | 32b 32b 32b 32b | * | 0 X X Y Y | 4 X Y 0 0 | * | 1 Z Z 0 0 | 5 X Y 0 0 | * | 2 X X Y Y | 6 X Y 0 0 | * | 3 Z Z 0 0 | 7 X Y 0 0 | * */ if (varying_has_user_specified_location(xfb_decl->matched_candidate->toplevel_var)) { output_size = MIN3(num_components, current_type_components_left, 4); current_type_components_left -= output_size; if (current_type_components_left == 0) { current_type_components_left = type_num_components; } } else { output_size = MIN2(num_components, 4 - location_frac); } assert((info->NumOutputs == 0 && max_outputs == 0) || info->NumOutputs < max_outputs); /* From the ARB_enhanced_layouts spec: * * "If such a block member or variable is not written during a shader * invocation, the buffer contents at the assigned offset will be * undefined. Even if there are no static writes to a variable or * member that is assigned a transform feedback offset, the space is * still allocated in the buffer and still affects the stride." */ if (xfb_decl_is_varying_written(xfb_decl)) { info->Outputs[info->NumOutputs].ComponentOffset = location_frac; info->Outputs[info->NumOutputs].OutputRegister = location; info->Outputs[info->NumOutputs].NumComponents = output_size; info->Outputs[info->NumOutputs].StreamId = xfb_decl->stream_id; info->Outputs[info->NumOutputs].OutputBuffer = buffer; info->Outputs[info->NumOutputs].DstOffset = xfb_offset; ++info->NumOutputs; } info->Buffers[buffer].Stream = xfb_decl->stream_id; xfb_offset += output_size; num_components -= output_size; location++; location_frac = 0; } } if (explicit_stride && explicit_stride[buffer]) { if (_mesa_gl_datatype_is_64bit(xfb_decl->type) && info->Buffers[buffer].Stride % 2) { linker_error(prog, "invalid qualifier xfb_stride=%d must be a " "multiple of 8 as its applied to a type that is or " "contains a double.", info->Buffers[buffer].Stride * 4); return false; } if (xfb_offset > info->Buffers[buffer].Stride) { linker_error(prog, "xfb_offset (%d) overflows xfb_stride (%d) for " "buffer (%d)", xfb_offset * 4, info->Buffers[buffer].Stride * 4, buffer); return false; } } else { if (max_member_alignment && has_xfb_qualifiers) { max_member_alignment[buffer] = MAX2(max_member_alignment[buffer], _mesa_gl_datatype_is_64bit(xfb_decl->type) ? 2 : 1); info->Buffers[buffer].Stride = ALIGN(xfb_offset, max_member_alignment[buffer]); } else { info->Buffers[buffer].Stride = xfb_offset; } } store_varying: info->Varyings[info->NumVarying].name.string = ralloc_strdup(prog, xfb_decl->orig_name); resource_name_updated(&info->Varyings[info->NumVarying].name); info->Varyings[info->NumVarying].Type = xfb_decl->type; info->Varyings[info->NumVarying].Size = size; info->Varyings[info->NumVarying].BufferIndex = buffer_index; info->NumVarying++; info->Buffers[buffer].NumVaryings++; return true; } static const struct tfeedback_candidate * xfb_decl_find_candidate(struct xfb_decl *xfb_decl, struct gl_shader_program *prog, struct hash_table *tfeedback_candidates) { const char *name = xfb_decl->var_name; switch (xfb_decl->lowered_builtin_array_variable) { case none: name = xfb_decl->var_name; break; case clip_distance: case cull_distance: name = "gl_ClipDistanceMESA"; break; } struct hash_entry *entry = _mesa_hash_table_search(tfeedback_candidates, name); xfb_decl->matched_candidate = entry ? (struct tfeedback_candidate *) entry->data : NULL; if (!xfb_decl->matched_candidate) { /* From GL_EXT_transform_feedback: * A program will fail to link if: * * * any variable name specified in the array is not * declared as an output in the geometry shader (if present) or * the vertex shader (if no geometry shader is present); */ linker_error(prog, "Transform feedback varying %s undeclared.", xfb_decl->orig_name); } return xfb_decl->matched_candidate; } /** * Force a candidate over the previously matched one. It happens when a new * varying needs to be created to match the xfb declaration, for example, * to fullfil an alignment criteria. */ static void xfb_decl_set_lowered_candidate(struct xfb_decl *xfb_decl, struct tfeedback_candidate *candidate) { xfb_decl->matched_candidate = candidate; /* The subscript part is no longer relevant */ xfb_decl->is_subscripted = false; xfb_decl->array_subscript = 0; } /** * Parse all the transform feedback declarations that were passed to * glTransformFeedbackVaryings() and store them in xfb_decl objects. * * If an error occurs, the error is reported through linker_error() and false * is returned. */ static bool parse_xfb_decls(const struct gl_constants *consts, const struct gl_extensions *exts, struct gl_shader_program *prog, const void *mem_ctx, unsigned num_names, char **varying_names, struct xfb_decl *decls, bool compact_arrays) { for (unsigned i = 0; i < num_names; ++i) { xfb_decl_init(&decls[i], consts, exts, mem_ctx, varying_names[i], compact_arrays); if (!xfb_decl_is_varying(&decls[i])) continue; /* From GL_EXT_transform_feedback: * A program will fail to link if: * * * any two entries in the array specify the same varying * variable; * * We interpret this to mean "any two entries in the array * specify the same varying variable and array index", since transform * feedback of arrays would be useless otherwise. */ for (unsigned j = 0; j < i; ++j) { if (xfb_decl_is_varying(&decls[j])) { if (xfb_decl_is_same(&decls[i], &decls[j])) { linker_error(prog, "Transform feedback varying %s specified " "more than once.", varying_names[i]); return false; } } } } return true; } static int cmp_xfb_offset(const void * x_generic, const void * y_generic) { struct xfb_decl *x = (struct xfb_decl *) x_generic; struct xfb_decl *y = (struct xfb_decl *) y_generic; if (x->buffer != y->buffer) return x->buffer - y->buffer; return x->offset - y->offset; } /** * Store transform feedback location assignments into * prog->sh.LinkedTransformFeedback based on the data stored in * xfb_decls. * * If an error occurs, the error is reported through linker_error() and false * is returned. */ static bool store_tfeedback_info(const struct gl_constants *consts, struct gl_shader_program *prog, unsigned num_xfb_decls, struct xfb_decl *xfb_decls, bool has_xfb_qualifiers, const void *mem_ctx) { if (!prog->last_vert_prog) return true; /* Make sure MaxTransformFeedbackBuffers is less than 32 so the bitmask for * tracking the number of buffers doesn't overflow. */ assert(consts->MaxTransformFeedbackBuffers < 32); bool separate_attribs_mode = prog->TransformFeedback.BufferMode == GL_SEPARATE_ATTRIBS; struct gl_program *xfb_prog = prog->last_vert_prog; xfb_prog->sh.LinkedTransformFeedback = rzalloc(xfb_prog, struct gl_transform_feedback_info); /* The xfb_offset qualifier does not have to be used in increasing order * however some drivers expect to receive the list of transform feedback * declarations in order so sort it now for convenience. */ if (has_xfb_qualifiers) { qsort(xfb_decls, num_xfb_decls, sizeof(*xfb_decls), cmp_xfb_offset); } xfb_prog->sh.LinkedTransformFeedback->Varyings = rzalloc_array(xfb_prog, struct gl_transform_feedback_varying_info, num_xfb_decls); unsigned num_outputs = 0; for (unsigned i = 0; i < num_xfb_decls; ++i) { if (xfb_decl_is_varying_written(&xfb_decls[i])) num_outputs += xfb_decl_get_num_outputs(&xfb_decls[i]); } xfb_prog->sh.LinkedTransformFeedback->Outputs = rzalloc_array(xfb_prog, struct gl_transform_feedback_output, num_outputs); unsigned num_buffers = 0; unsigned buffers = 0; BITSET_WORD *used_components[MAX_FEEDBACK_BUFFERS] = {0}; if (!has_xfb_qualifiers && separate_attribs_mode) { /* GL_SEPARATE_ATTRIBS */ for (unsigned i = 0; i < num_xfb_decls; ++i) { if (!xfb_decl_store(&xfb_decls[i], consts, prog, xfb_prog->sh.LinkedTransformFeedback, num_buffers, num_buffers, num_outputs, used_components, NULL, NULL, has_xfb_qualifiers, mem_ctx)) return false; buffers |= 1 << num_buffers; num_buffers++; } } else { /* GL_INVERLEAVED_ATTRIBS */ int buffer_stream_id = -1; unsigned buffer = num_xfb_decls ? xfb_decls[0].buffer : 0; bool explicit_stride[MAX_FEEDBACK_BUFFERS] = { false }; unsigned max_member_alignment[MAX_FEEDBACK_BUFFERS] = { 1, 1, 1, 1 }; /* Apply any xfb_stride global qualifiers */ if (has_xfb_qualifiers) { for (unsigned j = 0; j < MAX_FEEDBACK_BUFFERS; j++) { if (prog->TransformFeedback.BufferStride[j]) { explicit_stride[j] = true; xfb_prog->sh.LinkedTransformFeedback->Buffers[j].Stride = prog->TransformFeedback.BufferStride[j] / 4; } } } for (unsigned i = 0; i < num_xfb_decls; ++i) { if (has_xfb_qualifiers && buffer != xfb_decls[i].buffer) { /* we have moved to the next buffer so reset stream id */ buffer_stream_id = -1; num_buffers++; } if (xfb_decls[i].next_buffer_separator) { if (!xfb_decl_store(&xfb_decls[i], consts, prog, xfb_prog->sh.LinkedTransformFeedback, buffer, num_buffers, num_outputs, used_components, explicit_stride, max_member_alignment, has_xfb_qualifiers, mem_ctx)) return false; num_buffers++; buffer_stream_id = -1; continue; } if (has_xfb_qualifiers) { buffer = xfb_decls[i].buffer; } else { buffer = num_buffers; } if (xfb_decl_is_varying(&xfb_decls[i])) { if (buffer_stream_id == -1) { /* First varying writing to this buffer: remember its stream */ buffer_stream_id = (int) xfb_decls[i].stream_id; /* Only mark a buffer as active when there is a varying * attached to it. This behaviour is based on a revised version * of section 13.2.2 of the GL 4.6 spec. */ buffers |= 1 << buffer; } else if (buffer_stream_id != (int) xfb_decls[i].stream_id) { /* Varying writes to the same buffer from a different stream */ linker_error(prog, "Transform feedback can't capture varyings belonging " "to different vertex streams in a single buffer. " "Varying %s writes to buffer from stream %u, other " "varyings in the same buffer write from stream %u.", xfb_decls[i].orig_name, xfb_decls[i].stream_id, buffer_stream_id); return false; } } if (!xfb_decl_store(&xfb_decls[i], consts, prog, xfb_prog->sh.LinkedTransformFeedback, buffer, num_buffers, num_outputs, used_components, explicit_stride, max_member_alignment, has_xfb_qualifiers, mem_ctx)) return false; } } assert(xfb_prog->sh.LinkedTransformFeedback->NumOutputs == num_outputs); xfb_prog->sh.LinkedTransformFeedback->ActiveBuffers = buffers; return true; } /** * Enum representing the order in which varyings are packed within a * packing class. * * Currently we pack vec4's first, then vec2's, then scalar values, then * vec3's. This order ensures that the only vectors that are at risk of * having to be "double parked" (split between two adjacent varying slots) * are the vec3's. */ enum packing_order_enum { PACKING_ORDER_VEC4, PACKING_ORDER_VEC2, PACKING_ORDER_SCALAR, PACKING_ORDER_VEC3, }; /** * Structure recording the relationship between a single producer output * and a single consumer input. */ struct match { /** * Packing class for this varying, computed by compute_packing_class(). */ unsigned packing_class; /** * Packing order for this varying, computed by compute_packing_order(). */ enum packing_order_enum packing_order; /** * The output variable in the producer stage. */ nir_variable *producer_var; /** * The input variable in the consumer stage. */ nir_variable *consumer_var; /** * The location which has been assigned for this varying. This is * expressed in multiples of a float, with the first generic varying * (i.e. the one referred to by VARYING_SLOT_VAR0) represented by the * value 0. */ unsigned generic_location; /** * Original index, used as a fallback sorting key to ensure * a stable sort */ unsigned original_index; }; /** * Data structure recording the relationship between outputs of one shader * stage (the "producer") and inputs of another (the "consumer"). */ struct varying_matches { /** * If true, this driver disables varying packing, so all varyings need to * be aligned on slot boundaries, and take up a number of slots equal to * their number of matrix columns times their array size. * * Packing may also be disabled because our current packing method is not * safe in SSO or versions of OpenGL where interpolation qualifiers are not * guaranteed to match across stages. */ bool disable_varying_packing; /** * If true, this driver disables packing for varyings used by transform * feedback. */ bool disable_xfb_packing; /** * If true, this driver has transform feedback enabled. The transform * feedback code usually requires at least some packing be done even * when varying packing is disabled, fortunately where transform feedback * requires packing it's safe to override the disabled setting. See * is_varying_packing_safe(). */ bool xfb_enabled; bool enhanced_layouts_enabled; /** * If true, this driver prefers varyings to be aligned to power of two * in a slot. */ bool prefer_pot_aligned_varyings; struct match *matches; /** * The number of elements in the \c matches array that are currently in * use. */ unsigned num_matches; /** * The number of elements that were set aside for the \c matches array when * it was allocated. */ unsigned matches_capacity; gl_shader_stage producer_stage; gl_shader_stage consumer_stage; }; /** * Comparison function passed to qsort() to sort varyings by packing_class and * then by packing_order. */ static int varying_matches_match_comparator(const void *x_generic, const void *y_generic) { const struct match *x = (const struct match *) x_generic; const struct match *y = (const struct match *) y_generic; if (x->packing_class != y->packing_class) return x->packing_class - y->packing_class; if (x->packing_order != y->packing_order) return x->packing_order - y->packing_order; return x->original_index - y->original_index; } /** * Comparison function passed to qsort() to sort varyings used only by * transform feedback when packing of other varyings is disabled. */ static int varying_matches_xfb_comparator(const void *x_generic, const void *y_generic) { const struct match *x = (const struct match *) x_generic; const struct match *y = (const struct match *) y_generic; /* if both varying are used by transform feedback, sort them */ if (x->producer_var != NULL && x->producer_var->data.is_xfb_only) { if (y->producer_var != NULL && y->producer_var->data.is_xfb_only) return 0; /* if x is varying and y is not, put y first */ return +1; } else if (y->producer_var != NULL && y->producer_var->data.is_xfb_only) { /* if y is varying and x is not, leave x first */ return -1; } /* otherwise leave the order alone */ return x->original_index - y->original_index; } /** * Comparison function passed to qsort() to sort varyings NOT used by * transform feedback when packing of xfb varyings is disabled. */ static int varying_matches_not_xfb_comparator(const void *x_generic, const void *y_generic) { const struct match *x = (const struct match *) x_generic; const struct match *y = (const struct match *) y_generic; if ( (x->producer_var != NULL && !x->producer_var->data.is_xfb) && (y->producer_var != NULL && !y->producer_var->data.is_xfb) ) /* if both are non-xfb, then sort them */ return varying_matches_match_comparator(x_generic, y_generic); /* otherwise, leave the order alone */ return x->original_index - y->original_index; } static bool is_unpackable_tess(gl_shader_stage producer_stage, gl_shader_stage consumer_stage) { if (consumer_stage == MESA_SHADER_TESS_EVAL || consumer_stage == MESA_SHADER_TESS_CTRL || producer_stage == MESA_SHADER_TESS_CTRL) return true; return false; } static void init_varying_matches(void *mem_ctx, struct varying_matches *vm, const struct gl_constants *consts, const struct gl_extensions *exts, gl_shader_stage producer_stage, gl_shader_stage consumer_stage, bool sso) { /* Tessellation shaders treat inputs and outputs as shared memory and can * access inputs and outputs of other invocations. * Therefore, they can't be lowered to temps easily (and definitely not * efficiently). */ bool unpackable_tess = is_unpackable_tess(producer_stage, consumer_stage); /* Transform feedback code assumes varying arrays are packed, so if the * driver has disabled varying packing, make sure to at least enable * packing required by transform feedback. See below for exception. */ bool xfb_enabled = exts->EXT_transform_feedback && !unpackable_tess; /* Some drivers actually requires packing to be explicitly disabled * for varyings used by transform feedback. */ bool disable_xfb_packing = consts->DisableTransformFeedbackPacking; /* Disable packing on outward facing interfaces for SSO because in ES we * need to retain the unpacked varying information for draw time * validation. * * Packing is still enabled on individual arrays, structs, and matrices as * these are required by the transform feedback code and it is still safe * to do so. We also enable packing when a varying is only used for * transform feedback and its not a SSO. */ bool disable_varying_packing = consts->DisableVaryingPacking || unpackable_tess; if (sso && (producer_stage == MESA_SHADER_NONE || consumer_stage == MESA_SHADER_NONE)) disable_varying_packing = true; /* Note: this initial capacity is rather arbitrarily chosen to be large * enough for many cases without wasting an unreasonable amount of space. * varying_matches_record() will resize the array if there are more than * this number of varyings. */ vm->matches_capacity = 8; vm->matches = (struct match *) ralloc_array(mem_ctx, struct match, vm->matches_capacity); vm->num_matches = 0; vm->disable_varying_packing = disable_varying_packing; vm->disable_xfb_packing = disable_xfb_packing; vm->xfb_enabled = xfb_enabled; vm->enhanced_layouts_enabled = exts->ARB_enhanced_layouts; vm->prefer_pot_aligned_varyings = consts->PreferPOTAlignedVaryings; vm->producer_stage = producer_stage; vm->consumer_stage = consumer_stage; } /** * Packing is always safe on individual arrays, structures, and matrices. It * is also safe if the varying is only used for transform feedback. */ static bool is_varying_packing_safe(struct varying_matches *vm, const struct glsl_type *type, const nir_variable *var) { if (is_unpackable_tess(vm->producer_stage, vm->consumer_stage)) return false; return vm->xfb_enabled && (glsl_type_is_array_or_matrix(type) || glsl_type_is_struct(type) || var->data.is_xfb_only); } static bool is_packing_disabled(struct varying_matches *vm, const struct glsl_type *type, const nir_variable *var) { return (vm->disable_varying_packing && !is_varying_packing_safe(vm, type, var)) || (vm->disable_xfb_packing && var->data.is_xfb && !(glsl_type_is_array(type) || glsl_type_is_struct(type) || glsl_type_is_matrix(type))) || var->data.must_be_shader_input; } /** * Compute the "packing class" of the given varying. This is an unsigned * integer with the property that two variables in the same packing class can * be safely backed into the same vec4. */ static unsigned varying_matches_compute_packing_class(const nir_variable *var) { /* Without help from the back-end, there is no way to pack together * variables with different interpolation types, because * lower_packed_varyings must choose exactly one interpolation type for * each packed varying it creates. * * However, we can safely pack together floats, ints, and uints, because: * * - varyings of base type "int" and "uint" must use the "flat" * interpolation type, which can only occur in GLSL 1.30 and above. * * - On platforms that support GLSL 1.30 and above, lower_packed_varyings * can store flat floats as ints without losing any information (using * the ir_unop_bitcast_* opcodes). * * Therefore, the packing class depends only on the interpolation type. */ bool is_interpolation_flat = var->data.interpolation == INTERP_MODE_FLAT || glsl_contains_integer(var->type) || glsl_contains_double(var->type); const unsigned interp = is_interpolation_flat ? (unsigned) INTERP_MODE_FLAT : var->data.interpolation; assert(interp < (1 << 3)); const unsigned packing_class = (interp << 0) | (var->data.centroid << 3) | (var->data.sample << 4) | (var->data.patch << 5) | (var->data.must_be_shader_input << 6); return packing_class; } /** * Compute the "packing order" of the given varying. This is a sort key we * use to determine when to attempt to pack the given varying relative to * other varyings in the same packing class. */ static enum packing_order_enum varying_matches_compute_packing_order(const nir_variable *var) { const struct glsl_type *element_type = glsl_without_array(var->type); switch (glsl_get_component_slots(element_type) % 4) { case 1: return PACKING_ORDER_SCALAR; case 2: return PACKING_ORDER_VEC2; case 3: return PACKING_ORDER_VEC3; case 0: return PACKING_ORDER_VEC4; default: assert(!"Unexpected value of vector_elements"); return PACKING_ORDER_VEC4; } } /** * Record the given producer/consumer variable pair in the list of variables * that should later be assigned locations. * * It is permissible for \c consumer_var to be NULL (this happens if a * variable is output by the producer and consumed by transform feedback, but * not consumed by the consumer). * * If \c producer_var has already been paired up with a consumer_var, or * producer_var is part of fixed pipeline functionality (and hence already has * a location assigned), this function has no effect. * * Note: as a side effect this function may change the interpolation type of * \c producer_var, but only when the change couldn't possibly affect * rendering. */ static void varying_matches_record(void *mem_ctx, struct varying_matches *vm, nir_variable *producer_var, nir_variable *consumer_var) { assert(producer_var != NULL || consumer_var != NULL); if ((producer_var && (producer_var->data.explicit_location || producer_var->data.location != -1)) || (consumer_var && (consumer_var->data.explicit_location || consumer_var->data.location != -1))) { /* Either a location already exists for this variable (since it is part * of fixed functionality), or it has already been assigned explicitly. */ return; } /* The varyings should not have been matched and assgned previously */ assert((producer_var == NULL || producer_var->data.location == -1) && (consumer_var == NULL || consumer_var->data.location == -1)); bool needs_flat_qualifier = consumer_var == NULL && (glsl_contains_integer(producer_var->type) || glsl_contains_double(producer_var->type)); if (!vm->disable_varying_packing && (!vm->disable_xfb_packing || producer_var == NULL || !producer_var->data.is_xfb) && (needs_flat_qualifier || (vm->consumer_stage != MESA_SHADER_NONE && vm->consumer_stage != MESA_SHADER_FRAGMENT))) { /* Since this varying is not being consumed by the fragment shader, its * interpolation type varying cannot possibly affect rendering. * Also, this variable is non-flat and is (or contains) an integer * or a double. * If the consumer stage is unknown, don't modify the interpolation * type as it could affect rendering later with separate shaders. * * lower_packed_varyings requires all integer varyings to flat, * regardless of where they appear. We can trivially satisfy that * requirement by changing the interpolation type to flat here. */ if (producer_var) { producer_var->data.centroid = false; producer_var->data.sample = false; producer_var->data.interpolation = INTERP_MODE_FLAT; } if (consumer_var) { consumer_var->data.centroid = false; consumer_var->data.sample = false; consumer_var->data.interpolation = INTERP_MODE_FLAT; } } if (vm->num_matches == vm->matches_capacity) { vm->matches_capacity *= 2; vm->matches = (struct match *) reralloc(mem_ctx, vm->matches, struct match, vm->matches_capacity); } /* We must use the consumer to compute the packing class because in GL4.4+ * there is no guarantee interpolation qualifiers will match across stages. * * From Section 4.5 (Interpolation Qualifiers) of the GLSL 4.30 spec: * * "The type and presence of interpolation qualifiers of variables with * the same name declared in all linked shaders for the same cross-stage * interface must match, otherwise the link command will fail. * * When comparing an output from one stage to an input of a subsequent * stage, the input and output don't match if their interpolation * qualifiers (or lack thereof) are not the same." * * This text was also in at least revison 7 of the 4.40 spec but is no * longer in revision 9 and not in the 4.50 spec. */ const nir_variable *const var = (consumer_var != NULL) ? consumer_var : producer_var; if (producer_var && consumer_var && consumer_var->data.must_be_shader_input) { producer_var->data.must_be_shader_input = 1; } vm->matches[vm->num_matches].packing_class = varying_matches_compute_packing_class(var); vm->matches[vm->num_matches].packing_order = varying_matches_compute_packing_order(var); vm->matches[vm->num_matches].producer_var = producer_var; vm->matches[vm->num_matches].consumer_var = consumer_var; vm->num_matches++; } /** * Choose locations for all of the variable matches that were previously * passed to varying_matches_record(). * \param components returns array[slot] of number of components used * per slot (1, 2, 3 or 4) * \param reserved_slots bitmask indicating which varying slots are already * allocated * \return number of slots (4-element vectors) allocated */ static unsigned varying_matches_assign_locations(struct varying_matches *vm, struct gl_shader_program *prog, uint8_t components[], uint64_t reserved_slots) { /* Establish the original order of the varying_matches array; our * sorts will use this for sorting when the varyings do not have * xfb qualifiers */ for (unsigned i = 0; i < vm->num_matches; i++) vm->matches[i].original_index = i; /* If packing has been disabled then we cannot safely sort the varyings by * class as it may mean we are using a version of OpenGL where * interpolation qualifiers are not guaranteed to be matching across * shaders, sorting in this case could result in mismatching shader * interfaces. So we sort only the varyings used by transform feedback. * * If packing is only disabled for xfb varyings (mutually exclusive with * disable_varying_packing), we then group varyings depending on if they * are captured for transform feedback. */ if (vm->disable_varying_packing) { /* Only sort varyings that are only used by transform feedback. */ qsort(vm->matches, vm->num_matches, sizeof(*vm->matches), &varying_matches_xfb_comparator); } else if (vm->disable_xfb_packing) { /* Only sort varyings that are NOT used by transform feedback. */ qsort(vm->matches, vm->num_matches, sizeof(*vm->matches), &varying_matches_not_xfb_comparator); } else { /* Sort varying matches into an order that makes them easy to pack. */ qsort(vm->matches, vm->num_matches, sizeof(*vm->matches), &varying_matches_match_comparator); } unsigned generic_location = 0; unsigned generic_patch_location = MAX_VARYING*4; bool previous_var_xfb = false; bool previous_var_xfb_only = false; unsigned previous_packing_class = ~0u; /* For tranform feedback separate mode, we know the number of attributes * is <= the number of buffers. So packing isn't critical. In fact, * packing vec3 attributes can cause trouble because splitting a vec3 * effectively creates an additional transform feedback output. The * extra TFB output may exceed device driver limits. * * Also don't pack vec3 if the driver prefers power of two aligned * varyings. Packing order guarantees that vec4, vec2 and vec1 will be * pot-aligned, we only need to take care of vec3s */ const bool dont_pack_vec3 = (prog->TransformFeedback.BufferMode == GL_SEPARATE_ATTRIBS && prog->TransformFeedback.NumVarying > 0) || vm->prefer_pot_aligned_varyings; for (unsigned i = 0; i < vm->num_matches; i++) { unsigned *location = &generic_location; const nir_variable *var; const struct glsl_type *type; bool is_vertex_input = false; if (vm->matches[i].consumer_var) { var = vm->matches[i].consumer_var; type = get_varying_type(var, vm->consumer_stage); if (vm->consumer_stage == MESA_SHADER_VERTEX) is_vertex_input = true; } else { if (!vm->matches[i].producer_var) continue; /* The varying was optimised away */ var = vm->matches[i].producer_var; type = get_varying_type(var, vm->producer_stage); } if (var->data.patch) location = &generic_patch_location; /* Advance to the next slot if this varying has a different packing * class than the previous one, and we're not already on a slot * boundary. * * Also advance if varying packing is disabled for transform feedback, * and previous or current varying is used for transform feedback. * * Also advance to the next slot if packing is disabled. This makes sure * we don't assign varyings the same locations which is possible * because we still pack individual arrays, records and matrices even * when packing is disabled. Note we don't advance to the next slot if * we can pack varyings together that are only used for transform * feedback. */ if (var->data.must_be_shader_input || (vm->disable_xfb_packing && (previous_var_xfb || var->data.is_xfb)) || (vm->disable_varying_packing && !(previous_var_xfb_only && var->data.is_xfb_only)) || (previous_packing_class != vm->matches[i].packing_class) || (vm->matches[i].packing_order == PACKING_ORDER_VEC3 && dont_pack_vec3)) { *location = ALIGN(*location, 4); } previous_var_xfb = var->data.is_xfb; previous_var_xfb_only = var->data.is_xfb_only; previous_packing_class = vm->matches[i].packing_class; /* The number of components taken up by this variable. For vertex shader * inputs, we use the number of slots * 4, as they have different * counting rules. */ unsigned num_components = 0; if (is_vertex_input) { num_components = glsl_count_attribute_slots(type, is_vertex_input) * 4; } else { if (is_packing_disabled(vm, type, var)) { num_components = glsl_count_attribute_slots(type, false) * 4; } else { num_components = glsl_get_component_slots_aligned(type, *location); } } /* The last slot for this variable, inclusive. */ unsigned slot_end = *location + num_components - 1; /* FIXME: We could be smarter in the below code and loop back over * trying to fill any locations that we skipped because we couldn't pack * the varying between an explicit location. For now just let the user * hit the linking error if we run out of room and suggest they use * explicit locations. */ while (slot_end < MAX_VARYING * 4u) { const unsigned slots = (slot_end / 4u) - (*location / 4u) + 1; const uint64_t slot_mask = ((1ull << slots) - 1) << (*location / 4u); assert(slots > 0); if ((reserved_slots & slot_mask) == 0) { break; } *location = ALIGN(*location + 1, 4); slot_end = *location + num_components - 1; } if (!var->data.patch && slot_end >= MAX_VARYING * 4u) { linker_error(prog, "insufficient contiguous locations available for " "%s it is possible an array or struct could not be " "packed between varyings with explicit locations. Try " "using an explicit location for arrays and structs.", var->name); } if (slot_end < MAX_VARYINGS_INCL_PATCH * 4u) { for (unsigned j = *location / 4u; j < slot_end / 4u; j++) components[j] = 4; components[slot_end / 4u] = (slot_end & 3) + 1; } vm->matches[i].generic_location = *location; *location = slot_end + 1; } return (generic_location + 3) / 4; } static void varying_matches_assign_temp_locations(struct varying_matches *vm, struct gl_shader_program *prog, uint64_t reserved_slots) { unsigned tmp_loc = 0; for (unsigned i = 0; i < vm->num_matches; i++) { nir_variable *producer_var = vm->matches[i].producer_var; nir_variable *consumer_var = vm->matches[i].consumer_var; while (tmp_loc < MAX_VARYINGS_INCL_PATCH) { if (reserved_slots & (UINT64_C(1) << tmp_loc)) tmp_loc++; else break; } if (producer_var) { assert(producer_var->data.location == -1); producer_var->data.location = VARYING_SLOT_VAR0 + tmp_loc; } if (consumer_var) { assert(consumer_var->data.location == -1); consumer_var->data.location = VARYING_SLOT_VAR0 + tmp_loc; } tmp_loc++; } } /** * Update the producer and consumer shaders to reflect the locations * assignments that were made by varying_matches_assign_locations(). */ static void varying_matches_store_locations(struct varying_matches *vm) { /* Check is location needs to be packed with lower_packed_varyings() or if * we can just use ARB_enhanced_layouts packing. */ bool pack_loc[MAX_VARYINGS_INCL_PATCH] = {0}; const struct glsl_type *loc_type[MAX_VARYINGS_INCL_PATCH][4] = { {NULL, NULL} }; for (unsigned i = 0; i < vm->num_matches; i++) { nir_variable *producer_var = vm->matches[i].producer_var; nir_variable *consumer_var = vm->matches[i].consumer_var; unsigned generic_location = vm->matches[i].generic_location; unsigned slot = generic_location / 4; unsigned offset = generic_location % 4; if (producer_var) { producer_var->data.location = VARYING_SLOT_VAR0 + slot; producer_var->data.location_frac = offset; } if (consumer_var) { consumer_var->data.location = VARYING_SLOT_VAR0 + slot; consumer_var->data.location_frac = offset; } /* Find locations suitable for native packing via * ARB_enhanced_layouts. */ if (vm->enhanced_layouts_enabled) { nir_variable *var = producer_var ? producer_var : consumer_var; unsigned stage = producer_var ? vm->producer_stage : vm->consumer_stage; const struct glsl_type *type = get_varying_type(var, stage); unsigned comp_slots = glsl_get_component_slots(type) + offset; unsigned slots = comp_slots / 4; if (comp_slots % 4) slots += 1; if (producer_var && consumer_var) { if (glsl_type_is_array_or_matrix(type) || glsl_type_is_struct(type) || glsl_type_is_64bit(type)) { for (unsigned j = 0; j < slots; j++) { pack_loc[slot + j] = true; } } else if (offset + glsl_get_vector_elements(type) > 4) { pack_loc[slot] = true; pack_loc[slot + 1] = true; } else { loc_type[slot][offset] = type; } } else { for (unsigned j = 0; j < slots; j++) { pack_loc[slot + j] = true; } } } } /* Attempt to use ARB_enhanced_layouts for more efficient packing if * suitable. */ if (vm->enhanced_layouts_enabled) { for (unsigned i = 0; i < vm->num_matches; i++) { nir_variable *producer_var = vm->matches[i].producer_var; nir_variable *consumer_var = vm->matches[i].consumer_var; if (!producer_var || !consumer_var) continue; unsigned generic_location = vm->matches[i].generic_location; unsigned slot = generic_location / 4; if (pack_loc[slot]) continue; const struct glsl_type *type = get_varying_type(producer_var, vm->producer_stage); bool type_match = true; for (unsigned j = 0; j < 4; j++) { if (loc_type[slot][j]) { if (glsl_get_base_type(type) != glsl_get_base_type(loc_type[slot][j])) type_match = false; } } if (type_match) { producer_var->data.explicit_location = 1; consumer_var->data.explicit_location = 1; } } } } /** * Is the given variable a varying variable to be counted against the * limit in ctx->Const.MaxVarying? * This includes variables such as texcoords, colors and generic * varyings, but excludes variables such as gl_FrontFacing and gl_FragCoord. */ static bool var_counts_against_varying_limit(gl_shader_stage stage, const nir_variable *var) { /* Only fragment shaders will take a varying variable as an input */ if (stage == MESA_SHADER_FRAGMENT && var->data.mode == nir_var_shader_in) { switch (var->data.location) { case VARYING_SLOT_POS: case VARYING_SLOT_FACE: case VARYING_SLOT_PNTC: return false; default: return true; } } return false; } struct tfeedback_candidate_generator_state { /** * Memory context used to allocate hash table keys and values. */ void *mem_ctx; /** * Hash table in which tfeedback_candidate objects should be stored. */ struct hash_table *tfeedback_candidates; gl_shader_stage stage; /** * Pointer to the toplevel variable that is being traversed. */ nir_variable *toplevel_var; /** * Total number of varying floats that have been visited so far. This is * used to determine the offset to each varying within the toplevel * variable. */ unsigned varying_floats; /** * Offset within the xfb. Counted in floats. */ unsigned xfb_offset_floats; }; /** * Generates tfeedback_candidate structs describing all possible targets of * transform feedback. * * tfeedback_candidate structs are stored in the hash table * tfeedback_candidates. This hash table maps varying names to instances of the * tfeedback_candidate struct. */ static void tfeedback_candidate_generator(struct tfeedback_candidate_generator_state *state, char **name, size_t name_length, const struct glsl_type *type, const struct glsl_struct_field *named_ifc_member) { switch (glsl_get_base_type(type)) { case GLSL_TYPE_INTERFACE: if (named_ifc_member) { ralloc_asprintf_rewrite_tail(name, &name_length, ".%s", named_ifc_member->name); tfeedback_candidate_generator(state, name, name_length, named_ifc_member->type, NULL); return; } FALLTHROUGH; case GLSL_TYPE_STRUCT: for (unsigned i = 0; i < glsl_get_length(type); i++) { size_t new_length = name_length; /* Append '.field' to the current variable name. */ if (name) { ralloc_asprintf_rewrite_tail(name, &new_length, ".%s", glsl_get_struct_elem_name(type, i)); } tfeedback_candidate_generator(state, name, new_length, glsl_get_struct_field(type, i), NULL); } return; case GLSL_TYPE_ARRAY: if (glsl_type_is_struct(glsl_without_array(type)) || glsl_type_is_interface(glsl_without_array(type)) || glsl_type_is_array(glsl_get_array_element(type))) { for (unsigned i = 0; i < glsl_get_length(type); i++) { size_t new_length = name_length; /* Append the subscript to the current variable name */ ralloc_asprintf_rewrite_tail(name, &new_length, "[%u]", i); tfeedback_candidate_generator(state, name, new_length, glsl_get_array_element(type), named_ifc_member); } return; } FALLTHROUGH; default: assert(!glsl_type_is_struct(glsl_without_array(type))); assert(!glsl_type_is_interface(glsl_without_array(type))); struct tfeedback_candidate *candidate = rzalloc(state->mem_ctx, struct tfeedback_candidate); candidate->toplevel_var = state->toplevel_var; candidate->type = type; if (glsl_type_is_64bit(glsl_without_array(type))) { /* From ARB_gpu_shader_fp64: * * If any variable captured in transform feedback has double-precision * components, the practical requirements for defined behavior are: * ... * (c) each double-precision variable captured must be aligned to a * multiple of eight bytes relative to the beginning of a vertex. */ state->xfb_offset_floats = ALIGN(state->xfb_offset_floats, 2); /* 64-bit members of structs are also aligned. */ state->varying_floats = ALIGN(state->varying_floats, 2); } candidate->xfb_offset_floats = state->xfb_offset_floats; candidate->struct_offset_floats = state->varying_floats; _mesa_hash_table_insert(state->tfeedback_candidates, ralloc_strdup(state->mem_ctx, *name), candidate); const unsigned component_slots = glsl_get_component_slots(type); if (varying_has_user_specified_location(state->toplevel_var)) { state->varying_floats += glsl_count_attribute_slots(type, false) * 4; } else { state->varying_floats += component_slots; } state->xfb_offset_floats += component_slots; } } static void populate_consumer_input_sets(void *mem_ctx, nir_shader *nir, struct hash_table *consumer_inputs, struct hash_table *consumer_interface_inputs, nir_variable *consumer_inputs_with_locations[VARYING_SLOT_TESS_MAX]) { memset(consumer_inputs_with_locations, 0, sizeof(consumer_inputs_with_locations[0]) * VARYING_SLOT_TESS_MAX); nir_foreach_shader_in_variable(input_var, nir) { /* All interface blocks should have been lowered by this point */ assert(!glsl_type_is_interface(input_var->type)); if (input_var->data.explicit_location) { /* assign_varying_locations only cares about finding the * nir_variable at the start of a contiguous location block. * * - For !producer, consumer_inputs_with_locations isn't used. * * - For !consumer, consumer_inputs_with_locations is empty. * * For consumer && producer, if you were trying to set some * nir_variable to the middle of a location block on the other side * of producer/consumer, cross_validate_outputs_to_inputs() should * be link-erroring due to either type mismatch or location * overlaps. If the variables do match up, then they've got a * matching data.location and you only looked at * consumer_inputs_with_locations[var->data.location], not any * following entries for the array/structure. */ consumer_inputs_with_locations[input_var->data.location] = input_var; } else if (input_var->interface_type != NULL) { char *const iface_field_name = ralloc_asprintf(mem_ctx, "%s.%s", glsl_get_type_name(glsl_without_array(input_var->interface_type)), input_var->name); _mesa_hash_table_insert(consumer_interface_inputs, iface_field_name, input_var); } else { _mesa_hash_table_insert(consumer_inputs, ralloc_strdup(mem_ctx, input_var->name), input_var); } } } /** * Find a variable from the consumer that "matches" the specified variable * * This function only finds inputs with names that match. There is no * validation (here) that the types, etc. are compatible. */ static nir_variable * get_matching_input(void *mem_ctx, const nir_variable *output_var, struct hash_table *consumer_inputs, struct hash_table *consumer_interface_inputs, nir_variable *consumer_inputs_with_locations[VARYING_SLOT_TESS_MAX]) { nir_variable *input_var; if (output_var->data.explicit_location) { input_var = consumer_inputs_with_locations[output_var->data.location]; } else if (output_var->interface_type != NULL) { char *const iface_field_name = ralloc_asprintf(mem_ctx, "%s.%s", glsl_get_type_name(glsl_without_array(output_var->interface_type)), output_var->name); struct hash_entry *entry = _mesa_hash_table_search(consumer_interface_inputs, iface_field_name); input_var = entry ? (nir_variable *) entry->data : NULL; } else { struct hash_entry *entry = _mesa_hash_table_search(consumer_inputs, output_var->name); input_var = entry ? (nir_variable *) entry->data : NULL; } return (input_var == NULL || input_var->data.mode != nir_var_shader_in) ? NULL : input_var; } static int io_variable_cmp(const void *_a, const void *_b) { const nir_variable *const a = *(const nir_variable **) _a; const nir_variable *const b = *(const nir_variable **) _b; if (a->data.explicit_location && b->data.explicit_location) return b->data.location - a->data.location; if (a->data.explicit_location && !b->data.explicit_location) return 1; if (!a->data.explicit_location && b->data.explicit_location) return -1; return -strcmp(a->name, b->name); } /** * Sort the shader IO variables into canonical order */ static void canonicalize_shader_io(nir_shader *nir, nir_variable_mode io_mode) { nir_variable *var_table[MAX_PROGRAM_OUTPUTS * 4]; unsigned num_variables = 0; nir_foreach_variable_with_modes(var, nir, io_mode) { /* If we have already encountered more I/O variables that could * successfully link, bail. */ if (num_variables == ARRAY_SIZE(var_table)) return; var_table[num_variables++] = var; } if (num_variables == 0) return; /* Sort the list in reverse order (io_variable_cmp handles this). Later * we're going to push the variables on to the IR list as a stack, so we * want the last variable (in canonical order) to be first in the list. */ qsort(var_table, num_variables, sizeof(var_table[0]), io_variable_cmp); /* Remove the variable from it's current location in the varible list, and * put it at the front. */ for (unsigned i = 0; i < num_variables; i++) { exec_node_remove(&var_table[i]->node); exec_list_push_head(&nir->variables, &var_table[i]->node); } } /** * Generate a bitfield map of the explicit locations for shader varyings. * * Note: For Tessellation shaders we are sitting right on the limits of the * 64 bit map. Per-vertex and per-patch both have separate location domains * with a max of MAX_VARYING. */ static uint64_t reserved_varying_slot(struct gl_linked_shader *sh, nir_variable_mode io_mode) { assert(io_mode == nir_var_shader_in || io_mode == nir_var_shader_out); /* Avoid an overflow of the returned value */ assert(MAX_VARYINGS_INCL_PATCH <= 64); uint64_t slots = 0; int var_slot; if (!sh) return slots; nir_foreach_variable_with_modes(var, sh->Program->nir, io_mode) { if (!var->data.explicit_location || var->data.location < VARYING_SLOT_VAR0) continue; var_slot = var->data.location - VARYING_SLOT_VAR0; bool is_gl_vertex_input = io_mode == nir_var_shader_in && sh->Stage == MESA_SHADER_VERTEX; unsigned num_elements = glsl_count_attribute_slots(get_varying_type(var, sh->Stage), is_gl_vertex_input); for (unsigned i = 0; i < num_elements; i++) { if (var_slot >= 0 && var_slot < MAX_VARYINGS_INCL_PATCH) slots |= UINT64_C(1) << var_slot; var_slot += 1; } } return slots; } /** * Sets the bits in the inputs_read, or outputs_written * bitfield corresponding to this variable. */ static void set_variable_io_mask(BITSET_WORD *bits, nir_variable *var, gl_shader_stage stage) { assert(var->data.mode == nir_var_shader_in || var->data.mode == nir_var_shader_out); assert(var->data.location >= VARYING_SLOT_VAR0); const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, stage) || var->data.per_view) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } unsigned location = var->data.location - VARYING_SLOT_VAR0; unsigned slots = glsl_count_attribute_slots(type, false); for (unsigned i = 0; i < slots; i++) { BITSET_SET(bits, location + i); } } static uint8_t get_num_components(nir_variable *var) { if (glsl_type_is_struct_or_ifc(glsl_without_array(var->type))) return 4; return glsl_get_vector_elements(glsl_without_array(var->type)); } static void tcs_add_output_reads(nir_shader *shader, BITSET_WORD **read) { nir_foreach_function_impl(impl, shader) { nir_foreach_block(block, impl) { nir_foreach_instr(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (intrin->intrinsic != nir_intrinsic_load_deref) continue; nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (!nir_deref_mode_is(deref, nir_var_shader_out)) continue; nir_variable *var = nir_deref_instr_get_variable(deref); for (unsigned i = 0; i < get_num_components(var); i++) { if (var->data.location < VARYING_SLOT_VAR0) continue; unsigned comp = var->data.location_frac; set_variable_io_mask(read[comp + i], var, shader->info.stage); } } } } } /* We need to replace any interp intrinsics with undefined (shader_temp) inputs * as no further NIR pass expects to see this. */ static bool replace_unused_interpolate_at_with_undef(nir_builder *b, nir_instr *instr, void *data) { if (instr->type == nir_instr_type_intrinsic) { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (intrin->intrinsic == nir_intrinsic_interp_deref_at_centroid || intrin->intrinsic == nir_intrinsic_interp_deref_at_sample || intrin->intrinsic == nir_intrinsic_interp_deref_at_offset) { nir_variable *var = nir_intrinsic_get_var(intrin, 0); if (var->data.mode == nir_var_shader_temp) { /* Create undef and rewrite the interp uses */ nir_def *undef = nir_undef(b, intrin->def.num_components, intrin->def.bit_size); nir_def_replace(&intrin->def, undef); return true; } } } return false; } static void fixup_vars_lowered_to_temp(nir_shader *shader, nir_variable_mode mode) { /* Remove all interpolate uses of the unset varying and replace with undef. */ if (mode == nir_var_shader_in && shader->info.stage == MESA_SHADER_FRAGMENT) { (void) nir_shader_instructions_pass(shader, replace_unused_interpolate_at_with_undef, nir_metadata_control_flow, NULL); } nir_lower_global_vars_to_local(shader); nir_fixup_deref_modes(shader); } /** * Helper for removing unused shader I/O variables, by demoting them to global * variables (which may then be dead code eliminated). * * Example usage is: * * progress = nir_remove_unused_io_vars(producer, consumer, nir_var_shader_out, * read, patches_read) || * progress; * * The "used" should be an array of 4 BITSET_WORDs representing each * .location_frac used. Note that for vector variables, only the first channel * (.location_frac) is examined for deciding if the variable is used! */ static bool remove_unused_io_vars(nir_shader *producer, nir_shader *consumer, struct gl_shader_program *prog, nir_variable_mode mode, BITSET_WORD **used_by_other_stage) { assert(mode == nir_var_shader_in || mode == nir_var_shader_out); bool progress = false; nir_shader *shader = mode == nir_var_shader_out ? producer : consumer; BITSET_WORD **used; nir_foreach_variable_with_modes_safe(var, shader, mode) { used = used_by_other_stage; /* Skip builtins dead builtins are removed elsewhere */ if (is_gl_identifier(var->name)) continue; if (var->data.location < VARYING_SLOT_VAR0 && var->data.location >= 0) continue; /* Skip xfb varyings and any other type we cannot remove */ if (var->data.always_active_io) continue; if (var->data.explicit_xfb_buffer) continue; BITSET_WORD *other_stage = used[var->data.location_frac]; /* if location == -1 lower varying to global as it has no match and is not * a xfb varying, this must be done after skiping bultins as builtins * could be assigned a location of -1. * We also lower unused varyings with explicit locations. */ bool use_found = false; if (var->data.location >= 0) { unsigned location = var->data.location - VARYING_SLOT_VAR0; const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, shader->info.stage) || var->data.per_view) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } unsigned slots = glsl_count_attribute_slots(type, false); for (unsigned i = 0; i < slots; i++) { if (BITSET_TEST(other_stage, location + i)) { use_found = true; break; } } } if (!use_found) { /* This one is invalid, make it a global variable instead */ var->data.location = 0; var->data.mode = nir_var_shader_temp; progress = true; if (mode == nir_var_shader_in) { if (!prog->IsES && prog->GLSL_Version <= 120) { /* On page 25 (page 31 of the PDF) of the GLSL 1.20 spec: * * Only those varying variables used (i.e. read) in * the fragment shader executable must be written to * by the vertex shader executable; declaring * superfluous varying variables in a vertex shader is * permissible. * * We interpret this text as meaning that the VS must * write the variable for the FS to read it. See * "glsl1-varying read but not written" in piglit. */ linker_error(prog, "%s shader varying %s not written " "by %s shader\n.", _mesa_shader_stage_to_string(consumer->info.stage), var->name, _mesa_shader_stage_to_string(producer->info.stage)); } else { linker_warning(prog, "%s shader varying %s not written " "by %s shader\n.", _mesa_shader_stage_to_string(consumer->info.stage), var->name, _mesa_shader_stage_to_string(producer->info.stage)); } } } } if (progress) fixup_vars_lowered_to_temp(shader, mode); return progress; } static bool remove_unused_varyings(nir_shader *producer, nir_shader *consumer, struct gl_shader_program *prog, void *mem_ctx) { assert(producer->info.stage != MESA_SHADER_FRAGMENT); assert(consumer->info.stage != MESA_SHADER_VERTEX); int max_loc_out = 0; nir_foreach_shader_out_variable(var, producer) { if (var->data.location < VARYING_SLOT_VAR0) continue; const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, producer->info.stage) || var->data.per_view) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } unsigned slots = glsl_count_attribute_slots(type, false); max_loc_out = max_loc_out < (var->data.location - VARYING_SLOT_VAR0) + slots ? (var->data.location - VARYING_SLOT_VAR0) + slots : max_loc_out; } int max_loc_in = 0; nir_foreach_shader_in_variable(var, consumer) { if (var->data.location < VARYING_SLOT_VAR0) continue; const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, consumer->info.stage) || var->data.per_view) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } unsigned slots = glsl_count_attribute_slots(type, false); max_loc_in = max_loc_in < (var->data.location - VARYING_SLOT_VAR0) + slots ? (var->data.location - VARYING_SLOT_VAR0) + slots : max_loc_in; } /* Old glsl shaders that don't use explicit locations can contain greater * than 64 varyings before unused varyings are removed so we must count them * and make use of the BITSET macros to keep track of used slots. Once we * have removed these excess varyings we can make use of further nir varying * linking optimimisation passes. */ BITSET_WORD *read[4]; BITSET_WORD *written[4]; int max_loc = MAX2(max_loc_in, max_loc_out); for (unsigned i = 0; i < 4; i++) { read[i] = rzalloc_array(mem_ctx, BITSET_WORD, BITSET_WORDS(max_loc)); written[i] = rzalloc_array(mem_ctx, BITSET_WORD, BITSET_WORDS(max_loc)); } nir_foreach_shader_out_variable(var, producer) { if (var->data.location < VARYING_SLOT_VAR0) continue; for (unsigned i = 0; i < get_num_components(var); i++) { unsigned comp = var->data.location_frac; set_variable_io_mask(written[comp + i], var, producer->info.stage); } } nir_foreach_shader_in_variable(var, consumer) { if (var->data.location < VARYING_SLOT_VAR0) continue; for (unsigned i = 0; i < get_num_components(var); i++) { unsigned comp = var->data.location_frac; set_variable_io_mask(read[comp + i], var, consumer->info.stage); } } /* Each TCS invocation can read data written by other TCS invocations, * so even if the outputs are not used by the TES we must also make * sure they are not read by the TCS before demoting them to globals. */ if (producer->info.stage == MESA_SHADER_TESS_CTRL) tcs_add_output_reads(producer, read); bool progress = false; progress = remove_unused_io_vars(producer, consumer, prog, nir_var_shader_out, read); progress = remove_unused_io_vars(producer, consumer, prog, nir_var_shader_in, written) || progress; return progress; } static bool should_add_varying_match_record(nir_variable *const input_var, struct gl_shader_program *prog, struct gl_linked_shader *producer, struct gl_linked_shader *consumer) { /* If a matching input variable was found, add this output (and the input) to * the set. If this is a separable program and there is no consumer stage, * add the output. * * Always add TCS outputs. They are shared by all invocations * within a patch and can be used as shared memory. */ return input_var || (prog->SeparateShader && consumer == NULL) || producer->Stage == MESA_SHADER_TESS_CTRL; } /* This assigns some initial unoptimised varying locations so that our nir * optimisations can perform some initial optimisations and also does initial * processing of */ static bool assign_initial_varying_locations(const struct gl_constants *consts, const struct gl_extensions *exts, void *mem_ctx, struct gl_shader_program *prog, struct gl_linked_shader *producer, struct gl_linked_shader *consumer, unsigned num_xfb_decls, struct xfb_decl *xfb_decls, struct varying_matches *vm) { init_varying_matches(mem_ctx, vm, consts, exts, producer ? producer->Stage : MESA_SHADER_NONE, consumer ? consumer->Stage : MESA_SHADER_NONE, prog->SeparateShader); struct hash_table *tfeedback_candidates = _mesa_hash_table_create(mem_ctx, _mesa_hash_string, _mesa_key_string_equal); struct hash_table *consumer_inputs = _mesa_hash_table_create(mem_ctx, _mesa_hash_string, _mesa_key_string_equal); struct hash_table *consumer_interface_inputs = _mesa_hash_table_create(mem_ctx, _mesa_hash_string, _mesa_key_string_equal); nir_variable *consumer_inputs_with_locations[VARYING_SLOT_TESS_MAX] = { NULL, }; if (consumer) populate_consumer_input_sets(mem_ctx, consumer->Program->nir, consumer_inputs, consumer_interface_inputs, consumer_inputs_with_locations); if (producer) { nir_foreach_shader_out_variable(output_var, producer->Program->nir) { /* Only geometry shaders can use non-zero streams */ assert(output_var->data.stream == 0 || (output_var->data.stream < MAX_VERTEX_STREAMS && producer->Stage == MESA_SHADER_GEOMETRY)); if (num_xfb_decls > 0) { /* From OpenGL 4.6 (Core Profile) spec, section 11.1.2.1 * ("Vertex Shader Variables / Output Variables") * * "Each program object can specify a set of output variables from * one shader to be recorded in transform feedback mode (see * section 13.3). The variables that can be recorded are those * emitted by the first active shader, in order, from the * following list: * * * geometry shader * * tessellation evaluation shader * * tessellation control shader * * vertex shader" * * But on OpenGL ES 3.2, section 11.1.2.1 ("Vertex Shader * Variables / Output Variables") tessellation control shader is * not included in the stages list. */ if (!prog->IsES || producer->Stage != MESA_SHADER_TESS_CTRL) { const struct glsl_type *type = output_var->data.from_named_ifc_block ? output_var->interface_type : output_var->type; if (!output_var->data.patch && producer->Stage == MESA_SHADER_TESS_CTRL) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } const struct glsl_struct_field *ifc_member = NULL; if (output_var->data.from_named_ifc_block) { ifc_member = glsl_get_struct_field_data(glsl_without_array(type), glsl_get_field_index(glsl_without_array(type), output_var->name)); } char *name; if (glsl_type_is_struct(glsl_without_array(type)) || (glsl_type_is_array(type) && glsl_type_is_array(glsl_get_array_element(type)))) { type = output_var->type; name = ralloc_strdup(NULL, output_var->name); } else if (glsl_type_is_interface(glsl_without_array(type))) { name = ralloc_strdup(NULL, glsl_get_type_name(glsl_without_array(type))); } else { name = ralloc_strdup(NULL, output_var->name); } struct tfeedback_candidate_generator_state state; state.mem_ctx = mem_ctx; state.tfeedback_candidates = tfeedback_candidates; state.stage = producer->Stage; state.toplevel_var = output_var; state.varying_floats = 0; state.xfb_offset_floats = 0; tfeedback_candidate_generator(&state, &name, strlen(name), type, ifc_member); ralloc_free(name); } } nir_variable *const input_var = get_matching_input(mem_ctx, output_var, consumer_inputs, consumer_interface_inputs, consumer_inputs_with_locations); if (should_add_varying_match_record(input_var, prog, producer, consumer)) { varying_matches_record(mem_ctx, vm, output_var, input_var); } /* Only stream 0 outputs can be consumed in the next stage */ if (input_var && output_var->data.stream != 0) { linker_error(prog, "output %s is assigned to stream=%d but " "is linked to an input, which requires stream=0", output_var->name, output_var->data.stream); return false; } } } else { /* If there's no producer stage, then this must be a separable program. * For example, we may have a program that has just a fragment shader. * Later this program will be used with some arbitrary vertex (or * geometry) shader program. This means that locations must be assigned * for all the inputs. */ nir_foreach_shader_in_variable(input_var, consumer->Program->nir) { varying_matches_record(mem_ctx, vm, NULL, input_var); } } for (unsigned i = 0; i < num_xfb_decls; ++i) { if (!xfb_decl_is_varying(&xfb_decls[i])) continue; const struct tfeedback_candidate *matched_candidate = xfb_decl_find_candidate(&xfb_decls[i], prog, tfeedback_candidates); if (matched_candidate == NULL) return false; /* There are two situations where a new output varying is needed: * * - If varying packing is disabled for xfb and the current declaration * is subscripting an array, whether the subscript is aligned or not. * to preserve the rest of the array for the consumer. * * - If a builtin variable needs to be copied to a new variable * before its content is modified by another lowering pass (e.g. * \c gl_Position is transformed by \c nir_lower_viewport_transform). */ const bool lowered = (vm->disable_xfb_packing && xfb_decls[i].is_subscripted) || (matched_candidate->toplevel_var->data.explicit_location && matched_candidate->toplevel_var->data.location < VARYING_SLOT_VAR0 && (!consumer || consumer->Stage == MESA_SHADER_FRAGMENT) && (consts->ShaderCompilerOptions[producer->Stage].LowerBuiltinVariablesXfb & BITFIELD_BIT(matched_candidate->toplevel_var->data.location))); if (lowered) { nir_variable *new_var; struct tfeedback_candidate *new_candidate = NULL; new_var = gl_nir_lower_xfb_varying(producer->Program->nir, xfb_decls[i].orig_name, matched_candidate->toplevel_var); if (new_var == NULL) return false; /* Create new candidate and replace matched_candidate */ new_candidate = rzalloc(mem_ctx, struct tfeedback_candidate); new_candidate->toplevel_var = new_var; new_candidate->type = new_var->type; new_candidate->struct_offset_floats = 0; new_candidate->xfb_offset_floats = 0; _mesa_hash_table_insert(tfeedback_candidates, ralloc_strdup(mem_ctx, new_var->name), new_candidate); xfb_decl_set_lowered_candidate(&xfb_decls[i], new_candidate); matched_candidate = new_candidate; } /* Mark as xfb varying */ matched_candidate->toplevel_var->data.is_xfb = 1; /* Mark xfb varyings as always active */ matched_candidate->toplevel_var->data.always_active_io = 1; /* Mark any corresponding inputs as always active also. We must do this * because we have a NIR pass that lowers vectors to scalars and another * that removes unused varyings. * We don't split varyings marked as always active because there is no * point in doing so. This means we need to mark both sides of the * interface as always active otherwise we will have a mismatch and * start removing things we shouldn't. */ nir_variable *const input_var = get_matching_input(mem_ctx, matched_candidate->toplevel_var, consumer_inputs, consumer_interface_inputs, consumer_inputs_with_locations); if (input_var) { input_var->data.is_xfb = 1; input_var->data.always_active_io = 1; } /* Add the xfb varying to varying matches if it wasn't already added */ if ((!should_add_varying_match_record(input_var, prog, producer, consumer) && !matched_candidate->toplevel_var->data.is_xfb_only) || lowered) { matched_candidate->toplevel_var->data.is_xfb_only = 1; varying_matches_record(mem_ctx, vm, matched_candidate->toplevel_var, NULL); } } uint64_t reserved_out_slots = 0; if (producer) reserved_out_slots = reserved_varying_slot(producer, nir_var_shader_out); uint64_t reserved_in_slots = 0; if (consumer) reserved_in_slots = reserved_varying_slot(consumer, nir_var_shader_in); /* Assign temporary user varying locations. This is required for our NIR * varying optimisations to do their matching. */ const uint64_t reserved_slots = reserved_out_slots | reserved_in_slots; varying_matches_assign_temp_locations(vm, prog, reserved_slots); for (unsigned i = 0; i < num_xfb_decls; ++i) { if (!xfb_decl_is_varying(&xfb_decls[i])) continue; xfb_decls[i].matched_candidate->initial_location = xfb_decls[i].matched_candidate->toplevel_var->data.location; xfb_decls[i].matched_candidate->initial_location_frac = xfb_decls[i].matched_candidate->toplevel_var->data.location_frac; } return true; } static void link_shader_opts(struct varying_matches *vm, nir_shader *producer, nir_shader *consumer, struct gl_shader_program *prog, void *mem_ctx) { /* If we can't pack the stage using this pass then we can't lower io to * scalar just yet. Instead we leave it to a later NIR linking pass that uses * ARB_enhanced_layout style packing to pack things further. * * Otherwise we might end up causing linking errors and perf regressions * because the new scalars will be assigned individual slots and can overflow * the available slots. */ if (producer->options->lower_to_scalar && !vm->disable_varying_packing && !vm->disable_xfb_packing) { NIR_PASS(_, producer, nir_lower_io_to_scalar_early, nir_var_shader_out); NIR_PASS(_, consumer, nir_lower_io_to_scalar_early, nir_var_shader_in); } gl_nir_opts(producer); gl_nir_opts(consumer); if (nir_link_opt_varyings(producer, consumer)) gl_nir_opts(consumer); NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_out, NULL); NIR_PASS(_, consumer, nir_remove_dead_variables, nir_var_shader_in, NULL); if (remove_unused_varyings(producer, consumer, prog, mem_ctx)) { NIR_PASS(_, producer, nir_lower_global_vars_to_local); NIR_PASS(_, consumer, nir_lower_global_vars_to_local); gl_nir_opts(producer); gl_nir_opts(consumer); /* Optimizations can cause varyings to become unused. * nir_compact_varyings() depends on all dead varyings being removed so * we need to call nir_remove_dead_variables() again here. */ NIR_PASS(_, producer, nir_remove_dead_variables, nir_var_shader_out, NULL); NIR_PASS(_, consumer, nir_remove_dead_variables, nir_var_shader_in, NULL); } nir_link_varying_precision(producer, consumer); } /** * Assign locations for all variables that are produced in one pipeline stage * (the "producer") and consumed in the next stage (the "consumer"). * * Variables produced by the producer may also be consumed by transform * feedback. * * \param num_xfb_decls is the number of declarations indicating * variables that may be consumed by transform feedback. * * \param xfb_decls is a pointer to an array of xfb_decl objects * representing the result of parsing the strings passed to * glTransformFeedbackVaryings(). assign_location() will be called for * each of these objects that matches one of the outputs of the * producer. * * When num_xfb_decls is nonzero, it is permissible for the consumer to * be NULL. In this case, varying locations are assigned solely based on the * requirements of transform feedback. */ static bool assign_final_varying_locations(const struct gl_constants *consts, const struct gl_extensions *exts, void *mem_ctx, struct gl_shader_program *prog, struct gl_linked_shader *producer, struct gl_linked_shader *consumer, unsigned num_xfb_decls, struct xfb_decl *xfb_decls, const uint64_t reserved_slots, struct varying_matches *vm) { init_varying_matches(mem_ctx, vm, consts, exts, producer ? producer->Stage : MESA_SHADER_NONE, consumer ? consumer->Stage : MESA_SHADER_NONE, prog->SeparateShader); /* Regather varying matches as we ran optimisations and the previous pointers * are no longer valid. */ if (producer) { nir_foreach_shader_out_variable(var_out, producer->Program->nir) { if (var_out->data.location < VARYING_SLOT_VAR0 || var_out->data.explicit_location) continue; if (vm->num_matches == vm->matches_capacity) { vm->matches_capacity *= 2; vm->matches = (struct match *) reralloc(mem_ctx, vm->matches, struct match, vm->matches_capacity); } vm->matches[vm->num_matches].packing_class = varying_matches_compute_packing_class(var_out); vm->matches[vm->num_matches].packing_order = varying_matches_compute_packing_order(var_out); vm->matches[vm->num_matches].producer_var = var_out; vm->matches[vm->num_matches].consumer_var = NULL; vm->num_matches++; } /* Regather xfb varyings too */ for (unsigned i = 0; i < num_xfb_decls; i++) { if (!xfb_decl_is_varying(&xfb_decls[i])) continue; /* Varying pointer was already reset */ if (xfb_decls[i].matched_candidate->initial_location == -1) continue; bool UNUSED is_reset = false; bool UNUSED no_outputs = true; nir_foreach_shader_out_variable(var_out, producer->Program->nir) { no_outputs = false; assert(var_out->data.location != -1); if (var_out->data.location == xfb_decls[i].matched_candidate->initial_location && var_out->data.location_frac == xfb_decls[i].matched_candidate->initial_location_frac) { xfb_decls[i].matched_candidate->toplevel_var = var_out; xfb_decls[i].matched_candidate->initial_location = -1; is_reset = true; break; } } assert(is_reset || no_outputs); } } bool found_match = false; if (consumer) { nir_foreach_shader_in_variable(var_in, consumer->Program->nir) { if (var_in->data.location < VARYING_SLOT_VAR0 || var_in->data.explicit_location) continue; found_match = false; for (unsigned i = 0; i < vm->num_matches; i++) { if (vm->matches[i].producer_var && (vm->matches[i].producer_var->data.location == var_in->data.location && vm->matches[i].producer_var->data.location_frac == var_in->data.location_frac)) { vm->matches[i].consumer_var = var_in; found_match = true; break; } } if (!found_match) { if (vm->num_matches == vm->matches_capacity) { vm->matches_capacity *= 2; vm->matches = (struct match *) reralloc(mem_ctx, vm->matches, struct match, vm->matches_capacity); } vm->matches[vm->num_matches].packing_class = varying_matches_compute_packing_class(var_in); vm->matches[vm->num_matches].packing_order = varying_matches_compute_packing_order(var_in); vm->matches[vm->num_matches].producer_var = NULL; vm->matches[vm->num_matches].consumer_var = var_in; vm->num_matches++; } } } uint8_t components[MAX_VARYINGS_INCL_PATCH] = {0}; const unsigned slots_used = varying_matches_assign_locations(vm, prog, components, reserved_slots); varying_matches_store_locations(vm); for (unsigned i = 0; i < num_xfb_decls; ++i) { if (xfb_decl_is_varying(&xfb_decls[i])) { if (!xfb_decl_assign_location(&xfb_decls[i], consts, prog, vm->disable_varying_packing, vm->xfb_enabled)) return false; } } if (producer) { gl_nir_lower_packed_varyings(consts, prog, mem_ctx, slots_used, components, nir_var_shader_out, 0, producer, vm->disable_varying_packing, vm->disable_xfb_packing, vm->xfb_enabled); nir_lower_pack(producer->Program->nir); } if (consumer) { unsigned consumer_vertices = 0; if (consumer && consumer->Stage == MESA_SHADER_GEOMETRY) consumer_vertices = consumer->Program->nir->info.gs.vertices_in; gl_nir_lower_packed_varyings(consts, prog, mem_ctx, slots_used, components, nir_var_shader_in, consumer_vertices, consumer, vm->disable_varying_packing, vm->disable_xfb_packing, vm->xfb_enabled); nir_lower_pack(consumer->Program->nir); } return true; } static bool check_against_output_limit(const struct gl_constants *consts, gl_api api, struct gl_shader_program *prog, struct gl_linked_shader *producer, unsigned num_explicit_locations) { unsigned output_vectors = num_explicit_locations; nir_foreach_shader_out_variable(var, producer->Program->nir) { if (!var->data.explicit_location && var_counts_against_varying_limit(producer->Stage, var)) { /* outputs for fragment shader can't be doubles */ output_vectors += glsl_count_attribute_slots(var->type, false); } } assert(producer->Stage != MESA_SHADER_FRAGMENT); unsigned max_output_components = consts->Program[producer->Stage].MaxOutputComponents; const unsigned output_components = output_vectors * 4; if (output_components > max_output_components) { if (api == API_OPENGLES2 || prog->IsES) linker_error(prog, "%s shader uses too many output vectors " "(%u > %u)\n", _mesa_shader_stage_to_string(producer->Stage), output_vectors, max_output_components / 4); else linker_error(prog, "%s shader uses too many output components " "(%u > %u)\n", _mesa_shader_stage_to_string(producer->Stage), output_components, max_output_components); return false; } return true; } static bool check_against_input_limit(const struct gl_constants *consts, gl_api api, struct gl_shader_program *prog, struct gl_linked_shader *consumer, unsigned num_explicit_locations) { unsigned input_vectors = num_explicit_locations; nir_foreach_shader_in_variable(var, consumer->Program->nir) { if (!var->data.explicit_location && var_counts_against_varying_limit(consumer->Stage, var)) { /* vertex inputs aren't varying counted */ input_vectors += glsl_count_attribute_slots(var->type, false); } } assert(consumer->Stage != MESA_SHADER_VERTEX); unsigned max_input_components = consts->Program[consumer->Stage].MaxInputComponents; const unsigned input_components = input_vectors * 4; if (input_components > max_input_components) { if (api == API_OPENGLES2 || prog->IsES) linker_error(prog, "%s shader uses too many input vectors " "(%u > %u)\n", _mesa_shader_stage_to_string(consumer->Stage), input_vectors, max_input_components / 4); else linker_error(prog, "%s shader uses too many input components " "(%u > %u)\n", _mesa_shader_stage_to_string(consumer->Stage), input_components, max_input_components); return false; } return true; } /* Lower unset/unused inputs/outputs */ static void remove_unused_shader_inputs_and_outputs(struct gl_shader_program *prog, unsigned stage, nir_variable_mode mode) { bool progress = false; nir_shader *shader = prog->_LinkedShaders[stage]->Program->nir; nir_foreach_variable_with_modes_safe(var, shader, mode) { if (!var->data.is_xfb_only && var->data.location == -1) { var->data.location = 0; var->data.mode = nir_var_shader_temp; progress = true; } } if (progress) fixup_vars_lowered_to_temp(shader, mode); } static bool link_varyings(struct gl_shader_program *prog, unsigned first, unsigned last, const struct gl_constants *consts, const struct gl_extensions *exts, gl_api api, void *mem_ctx) { bool has_xfb_qualifiers = false; unsigned num_xfb_decls = 0; char **varying_names = NULL; bool compact_arrays = false; struct xfb_decl *xfb_decls = NULL; if (last > MESA_SHADER_FRAGMENT) return true; /* From the ARB_enhanced_layouts spec: * * "If the shader used to record output variables for transform feedback * varyings uses the "xfb_buffer", "xfb_offset", or "xfb_stride" layout * qualifiers, the values specified by TransformFeedbackVaryings are * ignored, and the set of variables captured for transform feedback is * instead derived from the specified layout qualifiers." */ for (int i = MESA_SHADER_FRAGMENT - 1; i >= 0; i--) { /* Find last stage before fragment shader */ if (prog->_LinkedShaders[i]) { has_xfb_qualifiers = process_xfb_layout_qualifiers(mem_ctx, prog->_LinkedShaders[i], prog, &num_xfb_decls, &varying_names, &compact_arrays); break; } } if (!has_xfb_qualifiers) { num_xfb_decls = prog->TransformFeedback.NumVarying; varying_names = prog->TransformFeedback.VaryingNames; } if (num_xfb_decls != 0) { /* From GL_EXT_transform_feedback: * A program will fail to link if: * * * the specified by TransformFeedbackVaryingsEXT is * non-zero, but the program object has no vertex or geometry * shader; */ if (first >= MESA_SHADER_FRAGMENT) { linker_error(prog, "Transform feedback varyings specified, but " "no vertex, tessellation, or geometry shader is " "present.\n"); return false; } xfb_decls = rzalloc_array(mem_ctx, struct xfb_decl, num_xfb_decls); if (!parse_xfb_decls(consts, exts, prog, mem_ctx, num_xfb_decls, varying_names, xfb_decls, compact_arrays)) return false; } struct gl_linked_shader *linked_shader[MESA_SHADER_STAGES]; unsigned num_shaders = 0; for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { if (prog->_LinkedShaders[i]) linked_shader[num_shaders++] = prog->_LinkedShaders[i]; } struct varying_matches vm; if (last < MESA_SHADER_FRAGMENT && (num_xfb_decls != 0 || prog->SeparateShader)) { struct gl_linked_shader *producer = prog->_LinkedShaders[last]; if (!assign_initial_varying_locations(consts, exts, mem_ctx, prog, producer, NULL, num_xfb_decls, xfb_decls, &vm)) return false; } if (last <= MESA_SHADER_FRAGMENT && !prog->SeparateShader) { remove_unused_shader_inputs_and_outputs(prog, first, nir_var_shader_in); remove_unused_shader_inputs_and_outputs(prog, last, nir_var_shader_out); } if (prog->SeparateShader) { struct gl_linked_shader *consumer = linked_shader[0]; if (!assign_initial_varying_locations(consts, exts, mem_ctx, prog, NULL, consumer, 0, NULL, &vm)) return false; } if (num_shaders == 1) { /* Linking shaders also optimizes them. Separate shaders, compute shaders * and shaders with a fixed-func VS or FS that don't need linking are * optimized here. */ gl_nir_opts(linked_shader[0]->Program->nir); } else { /* Linking the stages in the opposite order (from fragment to vertex) * ensures that inter-shader outputs written to in an earlier stage * are eliminated if they are (transitively) not used in a later * stage. */ for (int i = num_shaders - 2; i >= 0; i--) { unsigned stage_num_xfb_decls = linked_shader[i + 1]->Stage == MESA_SHADER_FRAGMENT ? num_xfb_decls : 0; if (!assign_initial_varying_locations(consts, exts, mem_ctx, prog, linked_shader[i], linked_shader[i + 1], stage_num_xfb_decls, xfb_decls, &vm)) return false; /* Now that validation is done its safe to remove unused varyings. As * we have both a producer and consumer its safe to remove unused * varyings even if the program is a SSO because the stages are being * linked together i.e. we have a multi-stage SSO. */ link_shader_opts(&vm, linked_shader[i]->Program->nir, linked_shader[i + 1]->Program->nir, prog, mem_ctx); remove_unused_shader_inputs_and_outputs(prog, linked_shader[i]->Stage, nir_var_shader_out); remove_unused_shader_inputs_and_outputs(prog, linked_shader[i + 1]->Stage, nir_var_shader_in); } } if (!prog->SeparateShader) { /* If not SSO remove unused varyings from the first/last stage */ NIR_PASS(_, prog->_LinkedShaders[first]->Program->nir, nir_remove_dead_variables, nir_var_shader_in, NULL); NIR_PASS(_, prog->_LinkedShaders[last]->Program->nir, nir_remove_dead_variables, nir_var_shader_out, NULL); } else { /* Sort inputs / outputs into a canonical order. This is necessary so * that inputs / outputs of separable shaders will be assigned * predictable locations regardless of the order in which declarations * appeared in the shader source. */ if (first != MESA_SHADER_VERTEX) { canonicalize_shader_io(prog->_LinkedShaders[first]->Program->nir, nir_var_shader_in); } if (last != MESA_SHADER_FRAGMENT) { canonicalize_shader_io(prog->_LinkedShaders[last]->Program->nir, nir_var_shader_out); } } /* If there is no fragment shader we need to set transform feedback. * * For SSO we also need to assign output locations. We assign them here * because we need to do it for both single stage programs and multi stage * programs. */ if (last < MESA_SHADER_FRAGMENT && (num_xfb_decls != 0 || prog->SeparateShader)) { const uint64_t reserved_out_slots = reserved_varying_slot(prog->_LinkedShaders[last], nir_var_shader_out); if (!assign_final_varying_locations(consts, exts, mem_ctx, prog, prog->_LinkedShaders[last], NULL, num_xfb_decls, xfb_decls, reserved_out_slots, &vm)) return false; } if (prog->SeparateShader) { struct gl_linked_shader *const sh = prog->_LinkedShaders[first]; const uint64_t reserved_slots = reserved_varying_slot(sh, nir_var_shader_in); /* Assign input locations for SSO, output locations are already * assigned. */ if (!assign_final_varying_locations(consts, exts, mem_ctx, prog, NULL /* producer */, sh /* consumer */, 0 /* num_xfb_decls */, NULL /* xfb_decls */, reserved_slots, &vm)) return false; } if (num_shaders == 1) { gl_nir_opt_dead_builtin_varyings(consts, api, prog, NULL, linked_shader[0], 0, NULL); gl_nir_opt_dead_builtin_varyings(consts, api, prog, linked_shader[0], NULL, num_xfb_decls, xfb_decls); } else { /* Linking the stages in the opposite order (from fragment to vertex) * ensures that inter-shader outputs written to in an earlier stage * are eliminated if they are (transitively) not used in a later * stage. */ int next = last; for (int i = next - 1; i >= 0; i--) { if (prog->_LinkedShaders[i] == NULL && i != 0) continue; struct gl_linked_shader *const sh_i = prog->_LinkedShaders[i]; struct gl_linked_shader *const sh_next = prog->_LinkedShaders[next]; gl_nir_opt_dead_builtin_varyings(consts, api, prog, sh_i, sh_next, next == MESA_SHADER_FRAGMENT ? num_xfb_decls : 0, xfb_decls); const uint64_t reserved_out_slots = reserved_varying_slot(sh_i, nir_var_shader_out); const uint64_t reserved_in_slots = reserved_varying_slot(sh_next, nir_var_shader_in); if (!assign_final_varying_locations(consts, exts, mem_ctx, prog, sh_i, sh_next, next == MESA_SHADER_FRAGMENT ? num_xfb_decls : 0, xfb_decls, reserved_out_slots | reserved_in_slots, &vm)) return false; /* This must be done after all dead varyings are eliminated. */ if (sh_i != NULL) { unsigned slots_used = util_bitcount64(reserved_out_slots); if (!check_against_output_limit(consts, api, prog, sh_i, slots_used)) return false; } unsigned slots_used = util_bitcount64(reserved_in_slots); if (!check_against_input_limit(consts, api, prog, sh_next, slots_used)) return false; next = i; } } if (!store_tfeedback_info(consts, prog, num_xfb_decls, xfb_decls, has_xfb_qualifiers, mem_ctx)) return false; return prog->data->LinkStatus != LINKING_FAILURE; } bool gl_assign_attribute_or_color_locations(const struct gl_constants *consts, struct gl_shader_program *prog) { void *mem_ctx = ralloc_context(NULL); if (!assign_attribute_or_color_locations(mem_ctx, prog, consts, MESA_SHADER_VERTEX, true)) { ralloc_free(mem_ctx); return false; } if (!assign_attribute_or_color_locations(mem_ctx, prog, consts, MESA_SHADER_FRAGMENT, true)) { ralloc_free(mem_ctx); return false; } ralloc_free(mem_ctx); return true; } bool gl_nir_link_varyings(const struct gl_constants *consts, const struct gl_extensions *exts, gl_api api, struct gl_shader_program *prog) { void *mem_ctx = ralloc_context(NULL); unsigned first, last; MESA_TRACE_FUNC(); first = MESA_SHADER_STAGES; last = 0; /* We need to initialise the program resource list because the varying * packing pass my start inserting varyings onto the list. */ init_program_resource_list(prog); /* Determine first and last stage. */ for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { if (!prog->_LinkedShaders[i]) continue; if (first == MESA_SHADER_STAGES) first = i; last = i; } bool r = link_varyings(prog, first, last, consts, exts, api, mem_ctx); if (r) { for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) { if (!prog->_LinkedShaders[i]) continue; /* Check for transform feedback varyings specified via the API */ prog->_LinkedShaders[i]->Program->nir->info.has_transform_feedback_varyings = prog->TransformFeedback.NumVarying > 0; /* Check for transform feedback varyings specified in the Shader */ if (prog->last_vert_prog) { prog->_LinkedShaders[i]->Program->nir->info.has_transform_feedback_varyings |= prog->last_vert_prog->sh.LinkedTransformFeedback->NumVarying > 0; } } /* Assign NIR XFB info to the last stage before the fragment shader */ for (int stage = MESA_SHADER_FRAGMENT - 1; stage >= 0; stage--) { struct gl_linked_shader *sh = prog->_LinkedShaders[stage]; if (sh && stage != MESA_SHADER_TESS_CTRL) { sh->Program->nir->xfb_info = gl_to_nir_xfb_info(sh->Program->sh.LinkedTransformFeedback, sh->Program->nir); break; } } /* Lower IO and thoroughly optimize and compact varyings. */ gl_nir_lower_optimize_varyings(consts, prog, false); } ralloc_free(mem_ctx); return r; }