Subversion Repositories Kolibri OS

Rev

Go to most recent revision | Blame | Last modification | View Log | RSS feed

  1. /**************************************************************************
  2.  *
  3.  * Copyright 2010 VMware, Inc.
  4.  * All Rights Reserved.
  5.  *
  6.  * Permission is hereby granted, free of charge, to any person obtaining a
  7.  * copy of this software and associated documentation files (the
  8.  * "Software"), to deal in the Software without restriction, including
  9.  * without limitation the rights to use, copy, modify, merge, publish,
  10.  * distribute, sub license, and/or sell copies of the Software, and to
  11.  * permit persons to whom the Software is furnished to do so, subject to
  12.  * the following conditions:
  13.  *
  14.  * The above copyright notice and this permission notice (including the
  15.  * next paragraph) shall be included in all copies or substantial portions
  16.  * of the Software.
  17.  *
  18.  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19.  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20.  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21.  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22.  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23.  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24.  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25.  *
  26.  **************************************************************************/
  27.  
  28. #include "util/u_math.h"
  29. #include "util/u_memory.h"
  30. #include "util/u_prim.h"
  31. #include "draw/draw_context.h"
  32. #include "draw/draw_gs.h"
  33. #include "draw/draw_vbuf.h"
  34. #include "draw/draw_vertex.h"
  35. #include "draw/draw_pt.h"
  36. #include "draw/draw_prim_assembler.h"
  37. #include "draw/draw_vs.h"
  38. #include "draw/draw_llvm.h"
  39. #include "gallivm/lp_bld_init.h"
  40.  
  41.  
  42. struct llvm_middle_end {
  43.    struct draw_pt_middle_end base;
  44.    struct draw_context *draw;
  45.  
  46.    struct pt_emit *emit;
  47.    struct pt_so_emit *so_emit;
  48.    struct pt_fetch *fetch;
  49.    struct pt_post_vs *post_vs;
  50.  
  51.  
  52.    unsigned vertex_data_offset;
  53.    unsigned vertex_size;
  54.    unsigned input_prim;
  55.    unsigned opt;
  56.  
  57.    struct draw_llvm *llvm;
  58.    struct draw_llvm_variant *current_variant;
  59. };
  60.  
  61.  
  62. static void
  63. llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  64. {
  65.    struct draw_context *draw = fpme->draw;
  66.    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  67.    struct draw_gs_llvm_variant_key *key;
  68.    struct draw_gs_llvm_variant *variant = NULL;
  69.    struct draw_gs_llvm_variant_list_item *li;
  70.    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  71.    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  72.    unsigned i;
  73.  
  74.    key = draw_gs_llvm_make_variant_key(fpme->llvm, store);
  75.  
  76.    /* Search shader's list of variants for the key */
  77.    li = first_elem(&shader->variants);
  78.    while (!at_end(&shader->variants, li)) {
  79.       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  80.          variant = li->base;
  81.          break;
  82.       }
  83.       li = next_elem(li);
  84.    }
  85.  
  86.    if (variant) {
  87.       /* found the variant, move to head of global list (for LRU) */
  88.       move_to_head(&fpme->llvm->gs_variants_list,
  89.                    &variant->list_item_global);
  90.    }
  91.    else {
  92.       /* Need to create new variant */
  93.  
  94.       /* First check if we've created too many variants.  If so, free
  95.        * 25% of the LRU to avoid using too much memory.
  96.        */
  97.       if (fpme->llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
  98.          /*
  99.           * XXX: should we flush here ?
  100.           */
  101.          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
  102.             struct draw_gs_llvm_variant_list_item *item;
  103.             if (is_empty_list(&fpme->llvm->gs_variants_list)) {
  104.                break;
  105.             }
  106.             item = last_elem(&fpme->llvm->gs_variants_list);
  107.             assert(item);
  108.             assert(item->base);
  109.             draw_gs_llvm_destroy_variant(item->base);
  110.          }
  111.       }
  112.  
  113.       variant = draw_gs_llvm_create_variant(fpme->llvm, gs->info.num_outputs, key);
  114.  
  115.       if (variant) {
  116.          insert_at_head(&shader->variants, &variant->list_item_local);
  117.          insert_at_head(&fpme->llvm->gs_variants_list,
  118.                         &variant->list_item_global);
  119.          fpme->llvm->nr_gs_variants++;
  120.          shader->variants_cached++;
  121.       }
  122.    }
  123.  
  124.    gs->current_variant = variant;
  125. }
  126.  
  127. /**
  128.  * Prepare/validate middle part of the vertex pipeline.
  129.  * NOTE: if you change this function, also look at the non-LLVM
  130.  * function fetch_pipeline_prepare() for similar changes.
  131.  */
  132. static void
  133. llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
  134.                          unsigned in_prim,
  135.                          unsigned opt,
  136.                          unsigned *max_vertices )
  137. {
  138.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  139.    struct draw_context *draw = fpme->draw;
  140.    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
  141.    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  142.    const unsigned out_prim = gs ? gs->output_primitive :
  143.       u_assembled_prim(in_prim);
  144.  
  145.    /* Add one to num_outputs because the pipeline occasionally tags on
  146.     * an additional texcoord, eg for AA lines.
  147.     */
  148.    const unsigned nr = MAX2( vs->info.num_inputs,
  149.                              vs->info.num_outputs + 1 );
  150.  
  151.    fpme->input_prim = in_prim;
  152.    fpme->opt = opt;
  153.  
  154.    /* Always leave room for the vertex header whether we need it or
  155.     * not.  It's hard to get rid of it in particular because of the
  156.     * viewport code in draw_pt_post_vs.c.
  157.     */
  158.    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
  159.  
  160.  
  161.    draw_pt_post_vs_prepare( fpme->post_vs,
  162.                             draw->clip_xy,
  163.                             draw->clip_z,
  164.                             draw->clip_user,
  165.                             draw->guard_band_xy,
  166.                             draw->identity_viewport,
  167.                             draw->rasterizer->clip_halfz,
  168.                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
  169.  
  170.    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
  171.  
  172.    if (!(opt & PT_PIPELINE)) {
  173.       draw_pt_emit_prepare( fpme->emit,
  174.                             out_prim,
  175.                             max_vertices );
  176.  
  177.       *max_vertices = MAX2( *max_vertices, 4096 );
  178.    }
  179.    else {
  180.       /* limit max fetches by limiting max_vertices */
  181.       *max_vertices = 4096;
  182.    }
  183.  
  184.    /* return even number */
  185.    *max_vertices = *max_vertices & ~1;
  186.  
  187.    /* Find/create the vertex shader variant */
  188.    {
  189.       struct draw_llvm_variant_key *key;
  190.       struct draw_llvm_variant *variant = NULL;
  191.       struct draw_llvm_variant_list_item *li;
  192.       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
  193.       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
  194.       unsigned i;
  195.  
  196.       key = draw_llvm_make_variant_key(fpme->llvm, store);
  197.  
  198.       /* Search shader's list of variants for the key */
  199.       li = first_elem(&shader->variants);
  200.       while (!at_end(&shader->variants, li)) {
  201.          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  202.             variant = li->base;
  203.             break;
  204.          }
  205.          li = next_elem(li);
  206.       }
  207.  
  208.       if (variant) {
  209.          /* found the variant, move to head of global list (for LRU) */
  210.          move_to_head(&fpme->llvm->vs_variants_list,
  211.                       &variant->list_item_global);
  212.       }
  213.       else {
  214.          /* Need to create new variant */
  215.  
  216.          /* First check if we've created too many variants.  If so, free
  217.           * 25% of the LRU to avoid using too much memory.
  218.           */
  219.          if (fpme->llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
  220.             /*
  221.              * XXX: should we flush here ?
  222.              */
  223.             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 4; i++) {
  224.                struct draw_llvm_variant_list_item *item;
  225.                if (is_empty_list(&fpme->llvm->vs_variants_list)) {
  226.                   break;
  227.                }
  228.                item = last_elem(&fpme->llvm->vs_variants_list);
  229.                assert(item);
  230.                assert(item->base);
  231.                draw_llvm_destroy_variant(item->base);
  232.             }
  233.          }
  234.  
  235.          variant = draw_llvm_create_variant(fpme->llvm, nr, key);
  236.  
  237.          if (variant) {
  238.             insert_at_head(&shader->variants, &variant->list_item_local);
  239.             insert_at_head(&fpme->llvm->vs_variants_list,
  240.                            &variant->list_item_global);
  241.             fpme->llvm->nr_variants++;
  242.             shader->variants_cached++;
  243.          }
  244.       }
  245.  
  246.       fpme->current_variant = variant;
  247.    }
  248.  
  249.    if (gs) {
  250.       llvm_middle_end_prepare_gs(fpme);
  251.    }
  252. }
  253.  
  254.  
  255. /**
  256.  * Bind/update constant buffer pointers, clip planes and viewport dims.
  257.  * These are "light weight" parameters which aren't baked into the
  258.  * generated code.  Updating these items is much cheaper than revalidating
  259.  * and rebuilding the generated pipeline code.
  260.  */
  261. static void
  262. llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
  263. {
  264.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  265.    struct draw_context *draw = fpme->draw;
  266.    unsigned i;
  267.  
  268.    for (i = 0; i < Elements(fpme->llvm->jit_context.vs_constants); ++i) {
  269.       fpme->llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
  270.    }
  271.    for (i = 0; i < Elements(fpme->llvm->gs_jit_context.constants); ++i) {
  272.       fpme->llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
  273.    }
  274.  
  275.    fpme->llvm->jit_context.planes =
  276.       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
  277.    fpme->llvm->gs_jit_context.planes =
  278.       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
  279.  
  280.    fpme->llvm->jit_context.viewport = (float *) draw->viewports[0].scale;
  281.    fpme->llvm->gs_jit_context.viewport = (float *) draw->viewports[0].scale;
  282. }
  283.  
  284.  
  285. static void pipeline(struct llvm_middle_end *llvm,
  286.                      const struct draw_vertex_info *vert_info,
  287.                      const struct draw_prim_info *prim_info)
  288. {
  289.    if (prim_info->linear)
  290.       draw_pipeline_run_linear( llvm->draw,
  291.                                 vert_info,
  292.                                 prim_info);
  293.    else
  294.       draw_pipeline_run( llvm->draw,
  295.                          vert_info,
  296.                          prim_info );
  297. }
  298.  
  299. static void emit(struct pt_emit *emit,
  300.                  const struct draw_vertex_info *vert_info,
  301.                  const struct draw_prim_info *prim_info)
  302. {
  303.    if (prim_info->linear) {
  304.       draw_pt_emit_linear(emit, vert_info, prim_info);
  305.    }
  306.    else {
  307.       draw_pt_emit(emit, vert_info, prim_info);
  308.    }
  309. }
  310.  
  311. static void
  312. llvm_pipeline_generic( struct draw_pt_middle_end *middle,
  313.                        const struct draw_fetch_info *fetch_info,
  314.                        const struct draw_prim_info *in_prim_info )
  315. {
  316.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  317.    struct draw_context *draw = fpme->draw;
  318.    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
  319.    struct draw_prim_info gs_prim_info;
  320.    struct draw_vertex_info llvm_vert_info;
  321.    struct draw_vertex_info gs_vert_info;
  322.    struct draw_vertex_info *vert_info;
  323.    struct draw_prim_info ia_prim_info;
  324.    struct draw_vertex_info ia_vert_info;
  325.    const struct draw_prim_info *prim_info = in_prim_info;
  326.    boolean free_prim_info = FALSE;
  327.    unsigned opt = fpme->opt;
  328.    unsigned clipped = 0;
  329.  
  330.    llvm_vert_info.count = fetch_info->count;
  331.    llvm_vert_info.vertex_size = fpme->vertex_size;
  332.    llvm_vert_info.stride = fpme->vertex_size;
  333.    llvm_vert_info.verts =
  334.       (struct vertex_header *)MALLOC(fpme->vertex_size *
  335.                                      align(fetch_info->count,  lp_native_vector_width / 32));
  336.    if (!llvm_vert_info.verts) {
  337.       assert(0);
  338.       return;
  339.    }
  340.  
  341.    if (draw->collect_statistics) {
  342.       draw->statistics.ia_vertices += prim_info->count;
  343.       draw->statistics.ia_primitives +=
  344.          u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
  345.       draw->statistics.vs_invocations += fetch_info->count;
  346.    }
  347.  
  348.    if (fetch_info->linear)
  349.       clipped = fpme->current_variant->jit_func( &fpme->llvm->jit_context,
  350.                                        llvm_vert_info.verts,
  351.                                        draw->pt.user.vbuffer,
  352.                                        fetch_info->start,
  353.                                        fetch_info->count,
  354.                                        fpme->vertex_size,
  355.                                        draw->pt.vertex_buffer,
  356.                                        draw->instance_id);
  357.    else
  358.       clipped = fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
  359.                                             llvm_vert_info.verts,
  360.                                             draw->pt.user.vbuffer,
  361.                                             fetch_info->elts,
  362.                                             draw->pt.user.eltMax,
  363.                                             fetch_info->count,
  364.                                             fpme->vertex_size,
  365.                                             draw->pt.vertex_buffer,
  366.                                             draw->instance_id);
  367.  
  368.    /* Finished with fetch and vs:
  369.     */
  370.    fetch_info = NULL;
  371.    vert_info = &llvm_vert_info;
  372.  
  373.  
  374.    if ((opt & PT_SHADE) && gshader) {
  375.       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
  376.       draw_geometry_shader_run(gshader,
  377.                                draw->pt.user.gs_constants,
  378.                                draw->pt.user.gs_constants_size,
  379.                                vert_info,
  380.                                prim_info,
  381.                                &vshader->info,
  382.                                &gs_vert_info,
  383.                                &gs_prim_info);
  384.  
  385.       FREE(vert_info->verts);
  386.       vert_info = &gs_vert_info;
  387.       prim_info = &gs_prim_info;
  388.    } else {
  389.       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
  390.          draw_prim_assembler_run(draw, prim_info, vert_info,
  391.                                  &ia_prim_info, &ia_vert_info);
  392.  
  393.          if (ia_vert_info.count) {
  394.             FREE(vert_info->verts);
  395.             vert_info = &ia_vert_info;
  396.             prim_info = &ia_prim_info;
  397.             free_prim_info = TRUE;
  398.          }
  399.       }
  400.    }
  401.    if (prim_info->count == 0) {
  402.       debug_printf("GS/IA didn't emit any vertices!\n");
  403.      
  404.       FREE(vert_info->verts);
  405.       if (free_prim_info) {
  406.          FREE(prim_info->primitive_lengths);
  407.       }
  408.       return;
  409.    }
  410.  
  411.    /* stream output needs to be done before clipping */
  412.    draw_pt_so_emit( fpme->so_emit, vert_info, prim_info );
  413.  
  414.    draw_stats_clipper_primitives(draw, prim_info);
  415.  
  416.    /*
  417.     * if there's no position, need to stop now, or the latter stages
  418.     * will try to access non-existent position output.
  419.     */
  420.    if (draw_current_shader_position_output(draw) != -1) {
  421.       if ((opt & PT_SHADE) && gshader) {
  422.          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
  423.       }
  424.       if (clipped) {
  425.          opt |= PT_PIPELINE;
  426.       }
  427.  
  428.       /* Do we need to run the pipeline? Now will come here if clipped
  429.        */
  430.       if (opt & PT_PIPELINE) {
  431.          pipeline( fpme, vert_info, prim_info );
  432.       }
  433.       else {
  434.          emit( fpme->emit, vert_info, prim_info );
  435.       }
  436.    }
  437.    FREE(vert_info->verts);
  438.    if (free_prim_info) {
  439.       FREE(prim_info->primitive_lengths);
  440.    }
  441. }
  442.  
  443.  
  444. static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
  445.                                  const unsigned *fetch_elts,
  446.                                  unsigned fetch_count,
  447.                                  const ushort *draw_elts,
  448.                                  unsigned draw_count,
  449.                                  unsigned prim_flags )
  450. {
  451.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  452.    struct draw_fetch_info fetch_info;
  453.    struct draw_prim_info prim_info;
  454.  
  455.    fetch_info.linear = FALSE;
  456.    fetch_info.start = 0;
  457.    fetch_info.elts = fetch_elts;
  458.    fetch_info.count = fetch_count;
  459.  
  460.    prim_info.linear = FALSE;
  461.    prim_info.start = 0;
  462.    prim_info.count = draw_count;
  463.    prim_info.elts = draw_elts;
  464.    prim_info.prim = fpme->input_prim;
  465.    prim_info.flags = prim_flags;
  466.    prim_info.primitive_count = 1;
  467.    prim_info.primitive_lengths = &draw_count;
  468.  
  469.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  470. }
  471.  
  472.  
  473. static void llvm_middle_end_linear_run( struct draw_pt_middle_end *middle,
  474.                                        unsigned start,
  475.                                        unsigned count,
  476.                                        unsigned prim_flags)
  477. {
  478.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  479.    struct draw_fetch_info fetch_info;
  480.    struct draw_prim_info prim_info;
  481.  
  482.    fetch_info.linear = TRUE;
  483.    fetch_info.start = start;
  484.    fetch_info.count = count;
  485.    fetch_info.elts = NULL;
  486.  
  487.    prim_info.linear = TRUE;
  488.    prim_info.start = 0;
  489.    prim_info.count = count;
  490.    prim_info.elts = NULL;
  491.    prim_info.prim = fpme->input_prim;
  492.    prim_info.flags = prim_flags;
  493.    prim_info.primitive_count = 1;
  494.    prim_info.primitive_lengths = &count;
  495.  
  496.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  497. }
  498.  
  499.  
  500.  
  501. static boolean
  502. llvm_middle_end_linear_run_elts( struct draw_pt_middle_end *middle,
  503.                                  unsigned start,
  504.                                  unsigned count,
  505.                                  const ushort *draw_elts,
  506.                                  unsigned draw_count,
  507.                                  unsigned prim_flags )
  508. {
  509.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  510.    struct draw_fetch_info fetch_info;
  511.    struct draw_prim_info prim_info;
  512.  
  513.    fetch_info.linear = TRUE;
  514.    fetch_info.start = start;
  515.    fetch_info.count = count;
  516.    fetch_info.elts = NULL;
  517.  
  518.    prim_info.linear = FALSE;
  519.    prim_info.start = 0;
  520.    prim_info.count = draw_count;
  521.    prim_info.elts = draw_elts;
  522.    prim_info.prim = fpme->input_prim;
  523.    prim_info.flags = prim_flags;
  524.    prim_info.primitive_count = 1;
  525.    prim_info.primitive_lengths = &draw_count;
  526.  
  527.    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
  528.  
  529.    return TRUE;
  530. }
  531.  
  532.  
  533.  
  534. static void llvm_middle_end_finish( struct draw_pt_middle_end *middle )
  535. {
  536.    /* nothing to do */
  537. }
  538.  
  539. static void llvm_middle_end_destroy( struct draw_pt_middle_end *middle )
  540. {
  541.    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
  542.  
  543.    if (fpme->fetch)
  544.       draw_pt_fetch_destroy( fpme->fetch );
  545.  
  546.    if (fpme->emit)
  547.       draw_pt_emit_destroy( fpme->emit );
  548.  
  549.    if (fpme->so_emit)
  550.       draw_pt_so_emit_destroy( fpme->so_emit );
  551.  
  552.    if (fpme->post_vs)
  553.       draw_pt_post_vs_destroy( fpme->post_vs );
  554.  
  555.    FREE(middle);
  556. }
  557.  
  558.  
  559. struct draw_pt_middle_end *
  560. draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
  561. {
  562.    struct llvm_middle_end *fpme = 0;
  563.  
  564.    if (!draw->llvm)
  565.       return NULL;
  566.  
  567.    fpme = CALLOC_STRUCT( llvm_middle_end );
  568.    if (!fpme)
  569.       goto fail;
  570.  
  571.    fpme->base.prepare         = llvm_middle_end_prepare;
  572.    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
  573.    fpme->base.run             = llvm_middle_end_run;
  574.    fpme->base.run_linear      = llvm_middle_end_linear_run;
  575.    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
  576.    fpme->base.finish          = llvm_middle_end_finish;
  577.    fpme->base.destroy         = llvm_middle_end_destroy;
  578.  
  579.    fpme->draw = draw;
  580.  
  581.    fpme->fetch = draw_pt_fetch_create( draw );
  582.    if (!fpme->fetch)
  583.       goto fail;
  584.  
  585.    fpme->post_vs = draw_pt_post_vs_create( draw );
  586.    if (!fpme->post_vs)
  587.       goto fail;
  588.  
  589.    fpme->emit = draw_pt_emit_create( draw );
  590.    if (!fpme->emit)
  591.       goto fail;
  592.  
  593.    fpme->so_emit = draw_pt_so_emit_create( draw );
  594.    if (!fpme->so_emit)
  595.       goto fail;
  596.  
  597.    fpme->llvm = draw->llvm;
  598.    if (!fpme->llvm)
  599.       goto fail;
  600.  
  601.    fpme->current_variant = NULL;
  602.  
  603.    return &fpme->base;
  604.  
  605.  fail:
  606.    if (fpme)
  607.       llvm_middle_end_destroy( &fpme->base );
  608.  
  609.    return NULL;
  610. }
  611.