CPUExecutionProvider
    -49 Add
    -49 LayerNormalization axis=-1 epsilon=9.999999747378752e-06 stash_type=1
    +49 SkipLayerNormalization epsilon=9.999999747378752e-06
    381 nodes, weights 8d2c7fd0b12a
    FuseSkipLayerNorm x49 7262f0e3
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x322 a8015e09

CUDAExecutionProvider
    -49 Add
    -49 LayerNormalization axis=-1 epsilon=9.999999747378752e-06 stash_type=1
    +49 SkipLayerNormalization epsilon=9.999999747378752e-06
    381 nodes, weights 8d2c7fd0b12a
    FuseSkipLayerNorm x49 7262f0e3
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x322 a8015e09

CoreMLExecutionProvider
    +120 MatMul
    +96 Add
    +96 Reshape
    +96 Slice
    +96 Transpose perm=(0, 2, 1, 3)
    +25 Mul
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    -1 ReduceL2 keepdims=1 noop_with_empty_axes=0
    +1 ReduceSum keepdims=1
    +1 Sqrt
    984 nodes, weights 37bcbe91a3df
    DecomposeAttention x336 9f572203
    DecomposeReduceL2 x3 595fd07f
    SplitLargeReduction x264 7dedb499
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x371 b9d4dec1

MIGraphXExecutionProvider
    +96 Reshape
    +96 Transpose perm=(0, 2, 1, 3)
    +48 MatMul
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Mul
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    +23 Add
    -2 ReduceSum keepdims=0 noop_with_empty_axes=0
    +2 ReduceSum keepdims=1 noop_with_empty_axes=0
    +1 Squeeze
    742 nodes, weights bfde5e39ba0f
    DecomposeAttention x312 c4301c07
    ElideMaskQueryAxis, DecomposeAttention, DecomposeAttention, _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x24 4927aaee
    KeepdimsMeanPool, _FloatMaskCount, _FloatMaskCount x5 06264236
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x4 43b4e59c
    _FloatMaskCount x1 90400fa4
    _Fp16TokenEmbedding x2 871722a1
    unstamped x394 dc0164e1

NvTensorRTRTXExecutionProvider
    +96 Reshape
    +96 Transpose perm=(0, 2, 1, 3)
    +48 MatMul
    +24 Add
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Mul
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    742 nodes, weights dc07801aae96
    DecomposeAttention x336 9f572203
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x396 04794a30

OpenVINOExecutionProvider
    +96 Reshape
    +96 Transpose perm=(0, 2, 1, 3)
    +48 MatMul
    +24 Add
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Mul
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    742 nodes, weights dc07801aae96
    DecomposeAttention x336 9f572203
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x396 04794a30

RKNPU static
    +432 Slice
    +336 Add
    +336 MatMul
    +96 Reshape
    +96 Transpose perm=(0, 2, 1, 3)
    +72 Mul
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Div
    +24 Erf
    -24 Gelu approximate=none
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    -1 Gather axis=0
    1845 nodes, weights bf79ba6449df
    contract {"dims":[{}],"embedding":"transformer.embeddings.word_embeddings.weight_fp16"}
    opset 19
    DecomposeAttention x336 9f8ae2f6
    DecomposeGelu x120 c9ce4d2f
    SplitLargeReduction x1152 388e0f77
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x1 ee8d996d
    unstamped x228 d573bde9

TensorrtExecutionProvider
    +96 Reshape
    +96 Transpose perm=(0, 2, 1, 3)
    +48 MatMul
    +24 Add
    -24 Attention is_causal=0 kv_num_heads=16 q_num_heads=16 qk_matmul_output_mode=0 scale=0.125 softcap=0.0
    +24 Mul
    +24 Softmax axis=-1
    +24 Transpose perm=(0, 1, 3, 2)
    742 nodes, weights dc07801aae96
    DecomposeAttention x336 9f572203
    _AdditivePadMask, _BroadcastMaskRebuild, _BroadcastMaskRebuild, _BroadcastMaskRebuild x5 988daec4
    _FloatMaskCount x3 97cd145e
    _Fp16TokenEmbedding x2 871722a1
    unstamped x396 04794a30
