Subversion Repositories Kolibri OS

Rev

Blame | Last modification | View Log | RSS feed

  1. /**************************************************************************
  2.  *
  3.  * Copyright 2010 VMware, Inc.
  4.  * All Rights Reserved.
  5.  *
  6.  * Permission is hereby granted, free of charge, to any person obtaining a
  7.  * copy of this software and associated documentation files (the
  8.  * "Software"), to deal in the Software without restriction, including
  9.  * without limitation the rights to use, copy, modify, merge, publish,
  10.  * distribute, sub license, and/or sell copies of the Software, and to
  11.  * permit persons to whom the Software is furnished to do so, subject to
  12.  * the following conditions:
  13.  *
  14.  * The above copyright notice and this permission notice (including the
  15.  * next paragraph) shall be included in all copies or substantial portions
  16.  * of the Software.
  17.  *
  18.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19.  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20.  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21.  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22.  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23.  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24.  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25.  *
  26.  **************************************************************************/
  27.  
  28. #include "util/u_math.h"
  29. #include "util/u_memory.h"
  30. #include "util/u_prim.h"
  31. #include "draw/draw_context.h"
  32. #include "draw/draw_gs.h"
  33. #include "draw/draw_vbuf.h"
  34. #include "draw/draw_vertex.h"
  35. #include "draw/draw_pt.h"
  36. #include "draw/draw_prim_assembler.h"
  37. #include "draw/draw_vs.h"
  38. #include "draw/draw_llvm.h"
  39. #include "gallivm/lp_bld_init.h"
  40.  
  41.  
  42. struct llvm_middle_end {
  43.    struct draw_pt_middle_end base;
  44.    struct draw_context *draw;
  45.  
  46.    struct pt_emit *emit;
  47.    struct pt_so_emit *so_emit;
  48.    struct pt_fetch *fetch;
  49.    struct pt_post_vs *post_vs;
  50.  
  51.  
  52.    unsigned vertex_data_offset;
  53.    unsigned vertex_size;
  54.    unsigned input_prim;
  55.    unsigned opt;
  56.  
  57.    struct draw_llvm *llvm;
  58.    struct draw_llvm_variant *current_variant;
  59. };
  60.  
  61.  
  62. /** cast wrapper */
  63. static INLINE struct llvm_middle_end *
  64. llvm_middle_end(struct draw_pt_middle_end *middle)
  65. {
  66.    return (struct llvm_middle_end *) middle;
  67. }
  68.  
  69.  
  70. static void
  71. llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  72. {
  73.    struct draw_context *draw = fpme->draw;
  74.    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  75.    struct draw_gs_llvm_variant_key *key;
  76.    struct draw_gs_llvm_variant *variant = NULL;
  77.    struct draw_gs_llvm_variant_list_item *li;
  78.    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  79.    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  80.    unsigned i;
  81.  
  82.    key = draw_gs_llvm_make_variant_key(fpme->llvm, store);
  83.  
  84.    /* Search shader's list of variants for the key */
  85.    li = first_elem(&shader->variants);
  86.    while (!at_end(&shader->variants, li)) {
  87.       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  88.          variant = li->base;
  89.          break;
  90.       }
  91.       li = next_elem(li);
  92.    }
  93.  
  94.    if (variant) {
  95.       /* found the variant, move to head of global list (for LRU) */
  96.       move_to_head(&fpme->llvm->gs_variants_list,
  97.                    &variant->list_item_global);
  98.    }
  99.    else {
  100.       /* Need to create new variant */
  101.  
  102.       /* First check if we've created too many variants.  If so, free
  103.        * 25% of the LRU to avoid using too much memory.
  104.        */
  105.       if (fpme->llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
  106.          /*
  107.           * XXX: should we flush here ?
  108.           */
  109.          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
  110.             struct draw_gs_llvm_variant_list_item *item;
  111.             if (is_empty_list(&fpme->llvm->gs_variants_list)) {
  112.                break;
  113.             }
  114.             item = last_elem(&fpme->llvm->gs_variants_list);
  115.             assert(item);
  116.             assert(item->base);
  117.             draw_gs_llvm_destroy_variant(item->base);
  118.          }
  119.       }
  120.  
  121.       variant = draw_gs_llvm_create_variant(fpme->llvm, gs->info.num_outputs, key);
  122.  
  123.       if (variant) {
  124.          insert_at_head(&shader->variants, &variant->list_item_local);
  125.          insert_at_head(&fpme->llvm->gs_variants_list,
  126.                         &variant->list_item_global);
  127.          fpme->llvm->nr_gs_variants++;
  128.          shader->variants_cached++;
  129.       }
  130.    }
  131.  
  132.    gs->current_variant = variant;
  133. }
  134.  
  135. /**
  136.  * Prepare/validate middle part of the vertex pipeline.
  137.  * NOTE: if you change this function, also look at the non-LLVM
  138.  * function fetch_pipeline_prepare() for similar changes.
  139.  */
  140. static void
  141. llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
  142.                          unsigned in_prim,
  143.                          unsigned opt,
  144.                          unsigned *max_vertices )
  145. {
  146.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  147.    struct draw_context *draw = fpme->draw;
  148.    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
  149.    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  150.    const unsigned out_prim = gs ? gs->output_primitive :
  151.       u_assembled_prim(in_prim);
  152.    unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
  153.                          out_prim == PIPE_PRIM_POINTS;
  154.    unsigned nr;
  155.  
  156.    fpme->input_prim = in_prim;
  157.    fpme->opt = opt;
  158.  
  159.    draw_pt_post_vs_prepare( fpme->post_vs,
  160.                             draw->clip_xy,
  161.                             draw->clip_z,
  162.                             draw->clip_user,
  163.                             point_clip ? draw->guard_band_points_xy :
  164.                                          draw->guard_band_xy,
  165.                             draw->bypass_viewport,
  166.                             draw->rasterizer->clip_halfz,
  167.                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
  168.  
  169.    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
  170.  
  171.    if (!(opt & PT_PIPELINE)) {
  172.       draw_pt_emit_prepare( fpme->emit, out_prim,
  173.                             max_vertices );
  174.  
  175.       *max_vertices = MAX2( *max_vertices, 4096 );
  176.    }
  177.    else {
  178.       /* limit max fetches by limiting max_vertices */
  179.       *max_vertices = 4096;
  180.    }
  181.  
  182.    /* Get the number of float[4] attributes per vertex.
  183.     * Note: this must be done after draw_pt_emit_prepare() since that
  184.     * can effect the vertex size.
  185.     */
  186.    nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
  187.  
  188.    /* Always leave room for the vertex header whether we need it or
  189.     * not.  It's hard to get rid of it in particular because of the
  190.     * viewport code in draw_pt_post_vs.c.
  191.     */
  192.    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
  193.  
  194.    /* return even number */
  195.    *max_vertices = *max_vertices & ~1;
  196.  
  197.    /* Find/create the vertex shader variant */
  198.    {
  199.       struct draw_llvm_variant_key *key;
  200.       struct draw_llvm_variant *variant = NULL;
  201.       struct draw_llvm_variant_list_item *li;
  202.       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
  203.       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
  204.       unsigned i;
  205.  
  206.       key = draw_llvm_make_variant_key(fpme->llvm, store);
  207.  
  208.       /* Search shader's list of variants for the key */
  209.       li = first_elem(&shader->variants);
  210.       while (!at_end(&shader->variants, li)) {
  211.          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  212.             variant = li->base;
  213.             break;
  214.          }
  215.          li = next_elem(li);
  216.       }
  217.  
  218.       if (variant) {
  219.          /* found the variant, move to head of global list (for LRU) */
  220.          move_to_head(&fpme->llvm->vs_variants_list,
  221.                       &variant->list_item_global);
  222.       }
  223.       else {
  224.          /* Need to create new variant */
  225.  
  226.          /* First check if we've created too many variants.  If so, free
  227.           * 25% of the LRU to avoid using too much memory.
  228.           */
  229.          if (fpme->llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
  230.             /*
  231.              * XXX: should we flush here ?
  232.              */
  233.             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
  234.                struct draw_llvm_variant_list_item *item;
  235.                if (is_empty_list(&fpme->llvm->vs_variants_list)) {
  236.                   break;
  237.                }
  238.                item = last_elem(&fpme->llvm->vs_variants_list);
  239.                assert(item);
  240.                assert(item->base);
  241.                draw_llvm_destroy_variant(item->base);
  242.             }
  243.          }
  244.  
  245.          variant = draw_llvm_create_variant(fpme->llvm, nr, key);
  246.  
  247.          if (variant) {
  248.             insert_at_head(&shader->variants, &variant->list_item_local);
  249.             insert_at_head(&fpme->llvm->vs_variants_list,
  250.                            &variant->list_item_global);
  251.             fpme->llvm->nr_variants++;
  252.             shader->variants_cached++;
  253.          }
  254.       }
  255.  
  256.       fpme->current_variant = variant;
  257.    }
  258.  
  259.    if (gs) {
  260.       llvm_middle_end_prepare_gs(fpme);
  261.    }
  262. }
  263.  
  264.  
  265. /**
  266.  * Bind/update constant buffer pointers, clip planes and viewport dims.
  267.  * These are "light weight" parameters which aren't baked into the
  268.  * generated code.  Updating these items is much cheaper than revalidating
  269.  * and rebuilding the generated pipeline code.
  270.  */
  271. static void
  272. llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
  273. {
  274.    static const float fake_const_buf[4];
  275.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  276.    struct draw_context *draw = fpme->draw;
  277.    struct draw_llvm *llvm = fpme->llvm;
  278.    unsigned i;
  279.  
  280.    for (i = 0; i < Elements(llvm->jit_context.vs_constants); ++i) {
  281.       int num_consts =
  282.          draw->pt.user.vs_constants_size[i] / (sizeof(float) * 4);
  283.       llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
  284.       llvm->jit_context.num_vs_constants[i] = num_consts;
  285.       if (num_consts == 0) {
  286.          llvm->jit_context.vs_constants[i] = fake_const_buf;
  287.       }
  288.    }
  289.    for (i = 0; i < Elements(llvm->gs_jit_context.constants); ++i) {
  290.       int num_consts =
  291.          draw->pt.user.gs_constants_size[i] / (sizeof(float) * 4);
  292.       llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
  293.       llvm->gs_jit_context.num_constants[i] = num_consts;
  294.       if (num_consts == 0) {
  295.          llvm->gs_jit_context.constants[i] = fake_const_buf;
  296.       }
  297.    }
  298.  
  299.    llvm->jit_context.planes =
  300.       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
  301.    llvm->gs_jit_context.planes =
  302.       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
  303.  
  304.    llvm->jit_context.viewports = draw->viewports;
  305.    llvm->gs_jit_context.viewports = draw->viewports;
  306. }
  307.  
  308.  
  309. static void
  310. pipeline(struct llvm_middle_end *llvm,
  311.          const struct draw_vertex_info *vert_info,
  312.          const struct draw_prim_info *prim_info)
  313. {
  314.    if (prim_info->linear)
  315.       draw_pipeline_run_linear( llvm->draw,
  316.                                 vert_info,
  317.                                 prim_info);
  318.    else
  319.       draw_pipeline_run( llvm->draw,
  320.                          vert_info,
  321.                          prim_info );
  322. }
  323.  
  324.  
  325. static void
  326. emit(struct pt_emit *emit,
  327.      const struct draw_vertex_info *vert_info,
  328.      const struct draw_prim_info *prim_info)
  329. {
  330.    if (prim_info->linear) {
  331.       draw_pt_emit_linear(emit, vert_info, prim_info);
  332.    }
  333.    else {
  334.       draw_pt_emit(emit, vert_info, prim_info);
  335.    }
  336. }
  337.  
  338.  
  339. static void
  340. llvm_pipeline_generic(struct draw_pt_middle_end *middle,
  341.                       const struct draw_fetch_info *fetch_info,
  342.                       const struct draw_prim_info *in_prim_info)
  343. {
  344.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  345.    struct draw_context *draw = fpme->draw;
  346.    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
  347.    struct draw_prim_info gs_prim_info;
  348.    struct draw_vertex_info llvm_vert_info;
  349.    struct draw_vertex_info gs_vert_info;
  350.    struct draw_vertex_info *vert_info;
  351.    struct draw_prim_info ia_prim_info;
  352.    struct draw_vertex_info ia_vert_info;
  353.    const struct draw_prim_info *prim_info = in_prim_info;
  354.    boolean free_prim_info = FALSE;
  355.    unsigned opt = fpme->opt;
  356.    unsigned clipped = 0;
  357.  
  358.    llvm_vert_info.count = fetch_info->count;
  359.    llvm_vert_info.vertex_size = fpme->vertex_size;
  360.    llvm_vert_info.stride = fpme->vertex_size;
  361.    llvm_vert_info.verts = (struct vertex_header *)
  362.       MALLOC(fpme->vertex_size *
  363.              align(fetch_info->count, lp_native_vector_width / 32));
  364.    if (!llvm_vert_info.verts) {
  365.       assert(0);
  366.       return;
  367.    }
  368.  
  369.    if (draw->collect_statistics) {
  370.       draw->statistics.ia_vertices += prim_info->count;
  371.       draw->statistics.ia_primitives +=
  372.          u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
  373.       draw->statistics.vs_invocations += fetch_info->count;
  374.    }
  375.  
  376.    if (fetch_info->linear)
  377.       clipped = fpme->current_variant->jit_func( &fpme->llvm->jit_context,
  378.                                        llvm_vert_info.verts,
  379.                                        draw->pt.user.vbuffer,
  380.                                        fetch_info->start,
  381.                                        fetch_info->count,
  382.                                        fpme->vertex_size,
  383.                                        draw->pt.vertex_buffer,
  384.                                        draw->instance_id,
  385.                                        draw->start_index,
  386.                                        draw->start_instance);
  387.    else
  388.       clipped = fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
  389.                                             llvm_vert_info.verts,
  390.                                             draw->pt.user.vbuffer,
  391.                                             fetch_info->elts,
  392.                                             draw->pt.user.eltMax,
  393.                                             fetch_info->count,
  394.                                             fpme->vertex_size,
  395.                                             draw->pt.vertex_buffer,
  396.                                             draw->instance_id,
  397.                                             draw->pt.user.eltBias,
  398.                                             draw->start_instance);
  399.  
  400.    /* Finished with fetch and vs:
  401.     */
  402.    fetch_info = NULL;
  403.    vert_info = &llvm_vert_info;
  404.  
  405.    if ((opt & PT_SHADE) && gshader) {
  406.       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
  407.       draw_geometry_shader_run(gshader,
  408.                                draw->pt.user.gs_constants,
  409.                                draw->pt.user.gs_constants_size,
  410.                                vert_info,
  411.                                prim_info,
  412.                                &vshader->info,
  413.                                &gs_vert_info,
  414.                                &gs_prim_info);
  415.  
  416.       FREE(vert_info->verts);
  417.       vert_info = &gs_vert_info;
  418.       prim_info = &gs_prim_info;
  419.    } else {
  420.       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
  421.          draw_prim_assembler_run(draw, prim_info, vert_info,
  422.                                  &ia_prim_info, &ia_vert_info);
  423.  
  424.          if (ia_vert_info.count) {
  425.             FREE(vert_info->verts);
  426.             vert_info = &ia_vert_info;
  427.             prim_info = &ia_prim_info;
  428.             free_prim_info = TRUE;
  429.          }
  430.       }
  431.    }
  432.    if (prim_info->count == 0) {
  433.       debug_printf("GS/IA didn't emit any vertices!\n");
  434.  
  435.       FREE(vert_info->verts);
  436.       if (free_prim_info) {
  437.          FREE(prim_info->primitive_lengths);
  438.       }
  439.       return;
  440.    }
  441.  
  442.    /* stream output needs to be done before clipping */
  443.    draw_pt_so_emit( fpme->so_emit, vert_info, prim_info );
  444.  
  445.    draw_stats_clipper_primitives(draw, prim_info);
  446.  
  447.    /*
  448.     * if there's no position, need to stop now, or the latter stages
  449.     * will try to access non-existent position output.
  450.     */
  451.    if (draw_current_shader_position_output(draw) != -1) {
  452.       if ((opt & PT_SHADE) && (gshader ||
  453.                                draw->vs.vertex_shader->info.writes_viewport_index)) {
  454.          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
  455.       }
  456.       if (clipped) {
  457.          opt |= PT_PIPELINE;
  458.       }
  459.  
  460.       /* Do we need to run the pipeline? Now will come here if clipped
  461.        */
  462.       if (opt & PT_PIPELINE) {
  463.          pipeline( fpme, vert_info, prim_info );
  464.       }
  465.       else {
  466.          emit( fpme->emit, vert_info, prim_info );
  467.       }
  468.    }
  469.    FREE(vert_info->verts);
  470.    if (free_prim_info) {
  471.       FREE(prim_info->primitive_lengths);
  472.    }
  473. }
  474.  
  475.  
  476. static void
  477. llvm_middle_end_run(struct draw_pt_middle_end *middle,
  478.                     const unsigned *fetch_elts,
  479.                     unsigned fetch_count,
  480.                     const ushort *draw_elts,
  481.                     unsigned draw_count,
  482.                     unsigned prim_flags)
  483. {
  484.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  485.    struct draw_fetch_info fetch_info;
  486.    struct draw_prim_info prim_info;
  487.  
  488.    fetch_info.linear = FALSE;
  489.    fetch_info.start = 0;
  490.    fetch_info.elts = fetch_elts;
  491.    fetch_info.count = fetch_count;
  492.  
  493.    prim_info.linear = FALSE;
  494.    prim_info.start = 0;
  495.    prim_info.count = draw_count;
  496.    prim_info.elts = draw_elts;
  497.    prim_info.prim = fpme->input_prim;
  498.    prim_info.flags = prim_flags;
  499.    prim_info.primitive_count = 1;
  500.    prim_info.primitive_lengths = &draw_count;
  501.  
  502.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  503. }
  504.  
  505.  
  506. static void
  507. llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
  508.                            unsigned start,
  509.                            unsigned count,
  510.                            unsigned prim_flags)
  511. {
  512.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  513.    struct draw_fetch_info fetch_info;
  514.    struct draw_prim_info prim_info;
  515.  
  516.    fetch_info.linear = TRUE;
  517.    fetch_info.start = start;
  518.    fetch_info.count = count;
  519.    fetch_info.elts = NULL;
  520.  
  521.    prim_info.linear = TRUE;
  522.    prim_info.start = 0;
  523.    prim_info.count = count;
  524.    prim_info.elts = NULL;
  525.    prim_info.prim = fpme->input_prim;
  526.    prim_info.flags = prim_flags;
  527.    prim_info.primitive_count = 1;
  528.    prim_info.primitive_lengths = &count;
  529.  
  530.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  531. }
  532.  
  533.  
  534. static boolean
  535. llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
  536.                                 unsigned start,
  537.                                 unsigned count,
  538.                                 const ushort *draw_elts,
  539.                                 unsigned draw_count,
  540.                                 unsigned prim_flags)
  541. {
  542.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  543.    struct draw_fetch_info fetch_info;
  544.    struct draw_prim_info prim_info;
  545.  
  546.    fetch_info.linear = TRUE;
  547.    fetch_info.start = start;
  548.    fetch_info.count = count;
  549.    fetch_info.elts = NULL;
  550.  
  551.    prim_info.linear = FALSE;
  552.    prim_info.start = 0;
  553.    prim_info.count = draw_count;
  554.    prim_info.elts = draw_elts;
  555.    prim_info.prim = fpme->input_prim;
  556.    prim_info.flags = prim_flags;
  557.    prim_info.primitive_count = 1;
  558.    prim_info.primitive_lengths = &draw_count;
  559.  
  560.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  561.  
  562.    return TRUE;
  563. }
  564.  
  565.  
  566. static void
  567. llvm_middle_end_finish(struct draw_pt_middle_end *middle)
  568. {
  569.    /* nothing to do */
  570. }
  571.  
  572.  
  573. static void
  574. llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
  575. {
  576.    struct llvm_middle_end *fpme = llvm_middle_end(middle);
  577.  
  578.    if (fpme->fetch)
  579.       draw_pt_fetch_destroy( fpme->fetch );
  580.  
  581.    if (fpme->emit)
  582.       draw_pt_emit_destroy( fpme->emit );
  583.  
  584.    if (fpme->so_emit)
  585.       draw_pt_so_emit_destroy( fpme->so_emit );
  586.  
  587.    if (fpme->post_vs)
  588.       draw_pt_post_vs_destroy( fpme->post_vs );
  589.  
  590.    FREE(middle);
  591. }
  592.  
  593.  
  594. struct draw_pt_middle_end *
  595. draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
  596. {
  597.    struct llvm_middle_end *fpme = 0;
  598.  
  599.    if (!draw->llvm)
  600.       return NULL;
  601.  
  602.    fpme = CALLOC_STRUCT( llvm_middle_end );
  603.    if (!fpme)
  604.       goto fail;
  605.  
  606.    fpme->base.prepare         = llvm_middle_end_prepare;
  607.    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
  608.    fpme->base.run             = llvm_middle_end_run;
  609.    fpme->base.run_linear      = llvm_middle_end_linear_run;
  610.    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
  611.    fpme->base.finish          = llvm_middle_end_finish;
  612.    fpme->base.destroy         = llvm_middle_end_destroy;
  613.  
  614.    fpme->draw = draw;
  615.  
  616.    fpme->fetch = draw_pt_fetch_create( draw );
  617.    if (!fpme->fetch)
  618.       goto fail;
  619.  
  620.    fpme->post_vs = draw_pt_post_vs_create( draw );
  621.    if (!fpme->post_vs)
  622.       goto fail;
  623.  
  624.    fpme->emit = draw_pt_emit_create( draw );
  625.    if (!fpme->emit)
  626.       goto fail;
  627.  
  628.    fpme->so_emit = draw_pt_so_emit_create( draw );
  629.    if (!fpme->so_emit)
  630.       goto fail;
  631.  
  632.    fpme->llvm = draw->llvm;
  633.    if (!fpme->llvm)
  634.       goto fail;
  635.  
  636.    fpme->current_variant = NULL;
  637.  
  638.    return &fpme->base;
  639.  
  640.  fail:
  641.    if (fpme)
  642.       llvm_middle_end_destroy( &fpme->base );
  643.  
  644.    return NULL;
  645. }
  646.