CPUExecutionProvider
    -25 Add
    -25 LayerNormalization axis=-1 epsilon=9.999999974752427e-07 stash_type=1
    +25 SkipLayerNormalization epsilon=9.999999974752427e-07
    184 nodes, weights 1773a9f9c8c3
    FuseSkipLayerNorm x25 1f146903
    _ConstantifyReshapeTarget x1 f786528a
    _ScalarGatherToSlice x1 60360a10
    unstamped x157 c063eb7a

CUDAExecutionProvider
    -25 LayerNormalization axis=-1 epsilon=9.999999974752427e-07 stash_type=1
    +25 SkipLayerNormalization epsilon=9.999999974752427e-07
    -24 Add
    +2 Reshape
    -1 Conv auto_pad=NOTSET dilations=(1, 1) group=1 pads=(0, 0, 0, 0) strides=(32, 32)
    +1 MatMul
    -1 Reshape allowzero=1
    -1 Transpose perm=(0, 2, 1)
    +1 Transpose perm=(0, 2, 1, 3)
    -1 Transpose perm=(0, 3, 1, 2)
    185 nodes, weights 957914dc16c5
    FuseSkipLayerNorm x25 dcf6f048
    PatchEmbedToMatMul, _ConstantifyReshapeTarget x5 132b0235
    _ScalarGatherToSlice x1 60360a10
    unstamped x154 88072a77

CoreMLExecutionProvider
    +52 Reshape
    +52 Transpose perm=(0, 2, 1, 3)
    +26 MatMul
    +14 Mul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    -1 ReduceL2 keepdims=1 noop_with_empty_axes=0
    +1 ReduceSum keepdims=1
    +1 Sqrt
    -1 Transpose perm=(0, 3, 1, 2)
    366 nodes, weights b95c00f29db9
    DecomposeAttention x169 e55b1e5b
    DecomposeReduceL2 x3 afe36182
    _ConstantifyReshapeTarget x1 f786528a
    _ScalarGatherToSlice x1 60360a10
    unstamped x192 10a29c2c

MIGraphXExecutionProvider
    +52 Reshape
    +52 Transpose perm=(0, 2, 1, 3)
    +26 MatMul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Mul
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    365 nodes, weights b95c00f29db9
    DecomposeAttention x169 e55b1e5b
    _ConstantifyReshapeTarget x1 f786528a
    _ScalarGatherToSlice x1 60360a10
    unstamped x194 2fb54a4e

NvTensorRTRTXExecutionProvider
    +54 Reshape
    +53 Transpose perm=(0, 2, 1, 3)
    +27 MatMul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Mul
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    +1 Add
    -1 Conv auto_pad=NOTSET dilations=(1, 1) group=1 pads=(0, 0, 0, 0) strides=(32, 32)
    -1 Reshape allowzero=1
    -1 Transpose perm=(0, 2, 1)
    -1 Transpose perm=(0, 3, 1, 2)
    366 nodes, weights 19de8836c127
    DecomposeAttention x169 e55b1e5b
    PatchEmbedToMatMul, _ConstantifyReshapeTarget x5 132b0235
    _ScalarGatherToSlice x1 60360a10
    unstamped x191 0eafaea6

OpenVINOExecutionProvider
    +54 Reshape
    +53 Transpose perm=(0, 2, 1, 3)
    +27 MatMul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Mul
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    +1 Add
    -1 Conv auto_pad=NOTSET dilations=(1, 1) group=1 pads=(0, 0, 0, 0) strides=(32, 32)
    -1 Reshape allowzero=1
    -1 Transpose perm=(0, 2, 1)
    -1 Transpose perm=(0, 3, 1, 2)
    366 nodes, weights 19de8836c127
    DecomposeAttention x169 e55b1e5b
    PatchEmbedToMatMul, _ConstantifyReshapeTarget x5 132b0235
    _ScalarGatherToSlice x1 60360a10
    unstamped x191 0eafaea6

RKNPU static
    +97 MatMul
    +84 Add
    +84 Slice
    +54 Reshape
    +53 Transpose perm=(0, 2, 1, 3)
    +38 Mul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Div
    +13 Erf
    -13 Gelu approximate=tanh
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    -1 Cast to=1
    -1 Conv auto_pad=NOTSET dilations=(1, 1) group=1 pads=(0, 0, 0, 0) strides=(32, 32)
    -1 Reshape allowzero=1
    +1 Sub
    -1 Transpose perm=(0, 2, 1)
    -1 Transpose perm=(0, 3, 1, 2)
    641 nodes, weights 5a3e4d97371d
    contract {"dims":[{}]}
    opset 19
    DecomposeAttention x169 e55b1e5b
    DecomposeGelu x65 5a4f24cb
    OpaqueZeroMul x1 3037fff5
    PatchEmbedToMatMul, _ConstantifyReshapeTarget x4 6fa42cac
    SplitLargeReduction x221 e1527d17
    SplitLargeReduction, PatchEmbedToMatMul, _ConstantifyReshapeTarget x17 03d5dda6
    _ScalarGatherToSlice x1 60360a10
    unstamped x163 b6aec873

TensorrtExecutionProvider
    +54 Reshape
    +53 Transpose perm=(0, 2, 1, 3)
    +27 MatMul
    -13 Attention is_causal=0 kv_num_heads=12 q_num_heads=12 qk_matmul_output_mode=0 softcap=0.0
    +13 Mul
    +13 Softmax axis=-1
    +13 Transpose perm=(0, 1, 3, 2)
    +1 Add
    -1 Conv auto_pad=NOTSET dilations=(1, 1) group=1 pads=(0, 0, 0, 0) strides=(32, 32)
    -1 Reshape allowzero=1
    -1 Transpose perm=(0, 2, 1)
    -1 Transpose perm=(0, 3, 1, 2)
    366 nodes, weights 19de8836c127
    DecomposeAttention x169 e55b1e5b
    PatchEmbedToMatMul, _ConstantifyReshapeTarget x5 132b0235
    _ScalarGatherToSlice x1 60360a10
    unstamped x191 0eafaea6
