diff --git "a/compiled/styletts2_text_predictor_256.mlmodelc/model.mil" "b/compiled/styletts2_text_predictor_256.mlmodelc/model.mil" new file mode 100644--- /dev/null +++ "b/compiled/styletts2_text_predictor_256.mlmodelc/model.mil" @@ -0,0 +1,752 @@ +program(1.0) +[buildInfo = dict, tensor>({{"coremlc-component-MIL", "3520.4.1"}, {"coremlc-version", "3520.5.1"}, {"coremltools-component-torch", "2.11.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "8.3.0"}})] +{ + func main(tensor style, tensor tokens) { + tensor var_20 = const()[name = tensor("op_20"), val = tensor(0x1.99999ap-3)]; + tensor x_1_axis_0 = const()[name = tensor("x_1_axis_0"), val = tensor(0)]; + tensor x_1_batch_dims_0 = const()[name = tensor("x_1_batch_dims_0"), val = tensor(0)]; + tensor x_1_validate_indices_0 = const()[name = tensor("x_1_validate_indices_0"), val = tensor(false)]; + tensor text_encoder_embedding_weight_to_fp16 = const()[name = tensor("text_encoder_embedding_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64)))]; + tensor tokens_to_int16_dtype_0 = const()[name = tensor("tokens_to_int16_dtype_0"), val = tensor("int16")]; + tensor tokens_to_int16 = cast(dtype = tokens_to_int16_dtype_0, x = tokens)[name = tensor("cast_64")]; + tensor x_1_cast_fp16_cast_uint16 = gather(axis = x_1_axis_0, batch_dims = x_1_batch_dims_0, indices = tokens_to_int16, validate_indices = x_1_validate_indices_0, x = text_encoder_embedding_weight_to_fp16)[name = tensor("x_1_cast_fp16_cast_uint16")]; + tensor input_1_perm_0 = const()[name = tensor("input_1_perm_0"), val = tensor([0, 2, 1])]; + tensor x_3_pad_type_0 = const()[name = tensor("x_3_pad_type_0"), val = tensor("custom")]; + tensor x_3_pad_0 = const()[name = tensor("x_3_pad_0"), val = tensor([2, 2])]; + tensor x_3_strides_0 = const()[name = tensor("x_3_strides_0"), val = tensor([1])]; + tensor x_3_dilations_0 = const()[name = tensor("x_3_dilations_0"), val = tensor([1])]; + tensor x_3_groups_0 = const()[name = tensor("x_3_groups_0"), val = tensor(1)]; + tensor weight_3_to_fp16 = const()[name = tensor("weight_3_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(182400)))]; + tensor text_encoder_cnn_0_0_bias_to_fp16 = const()[name = tensor("text_encoder_cnn_0_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(2803904)))]; + tensor input_1_cast_fp16 = transpose(perm = input_1_perm_0, x = x_1_cast_fp16_cast_uint16)[name = tensor("transpose_177")]; + tensor x_3_cast_fp16 = conv(bias = text_encoder_cnn_0_0_bias_to_fp16, dilations = x_3_dilations_0, groups = x_3_groups_0, pad = x_3_pad_0, pad_type = x_3_pad_type_0, strides = x_3_strides_0, weight = weight_3_to_fp16, x = input_1_cast_fp16)[name = tensor("x_3_cast_fp16")]; + tensor input_3_perm_0 = const()[name = tensor("input_3_perm_0"), val = tensor([0, -1, 1])]; + tensor x_5_axes_0 = const()[name = tensor("x_5_axes_0"), val = tensor([-1])]; + tensor text_encoder_cnn_0_1_gamma_to_fp16 = const()[name = tensor("text_encoder_cnn_0_1_gamma_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(2804992)))]; + tensor text_encoder_cnn_0_1_beta_to_fp16 = const()[name = tensor("text_encoder_cnn_0_1_beta_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(2806080)))]; + tensor var_18_to_fp16 = const()[name = tensor("op_18_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_3_cast_fp16 = transpose(perm = input_3_perm_0, x = x_3_cast_fp16)[name = tensor("transpose_176")]; + tensor x_5_cast_fp16 = layer_norm(axes = x_5_axes_0, beta = text_encoder_cnn_0_1_beta_to_fp16, epsilon = var_18_to_fp16, gamma = text_encoder_cnn_0_1_gamma_to_fp16, x = input_3_cast_fp16)[name = tensor("x_5_cast_fp16")]; + tensor input_5_perm_0 = const()[name = tensor("input_5_perm_0"), val = tensor([0, -1, 1])]; + tensor input_5_cast_fp16 = transpose(perm = input_5_perm_0, x = x_5_cast_fp16)[name = tensor("transpose_175")]; + tensor input_7_cast_fp16 = leaky_relu(alpha = var_20, x = input_5_cast_fp16)[name = tensor("input_7_cast_fp16")]; + tensor x_7_pad_type_0 = const()[name = tensor("x_7_pad_type_0"), val = tensor("custom")]; + tensor x_7_pad_0 = const()[name = tensor("x_7_pad_0"), val = tensor([2, 2])]; + tensor x_7_strides_0 = const()[name = tensor("x_7_strides_0"), val = tensor([1])]; + tensor x_7_dilations_0 = const()[name = tensor("x_7_dilations_0"), val = tensor([1])]; + tensor x_7_groups_0 = const()[name = tensor("x_7_groups_0"), val = tensor(1)]; + tensor weight_7_to_fp16 = const()[name = tensor("weight_7_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(2807168)))]; + tensor text_encoder_cnn_1_0_bias_to_fp16 = const()[name = tensor("text_encoder_cnn_1_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(5428672)))]; + tensor x_7_cast_fp16 = conv(bias = text_encoder_cnn_1_0_bias_to_fp16, dilations = x_7_dilations_0, groups = x_7_groups_0, pad = x_7_pad_0, pad_type = x_7_pad_type_0, strides = x_7_strides_0, weight = weight_7_to_fp16, x = input_7_cast_fp16)[name = tensor("x_7_cast_fp16")]; + tensor input_11_perm_0 = const()[name = tensor("input_11_perm_0"), val = tensor([0, -1, 1])]; + tensor x_9_axes_0 = const()[name = tensor("x_9_axes_0"), val = tensor([-1])]; + tensor text_encoder_cnn_1_1_gamma_to_fp16 = const()[name = tensor("text_encoder_cnn_1_1_gamma_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(5429760)))]; + tensor text_encoder_cnn_1_1_beta_to_fp16 = const()[name = tensor("text_encoder_cnn_1_1_beta_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(5430848)))]; + tensor input_11_cast_fp16 = transpose(perm = input_11_perm_0, x = x_7_cast_fp16)[name = tensor("transpose_174")]; + tensor x_9_cast_fp16 = layer_norm(axes = x_9_axes_0, beta = text_encoder_cnn_1_1_beta_to_fp16, epsilon = var_18_to_fp16, gamma = text_encoder_cnn_1_1_gamma_to_fp16, x = input_11_cast_fp16)[name = tensor("x_9_cast_fp16")]; + tensor input_13_perm_0 = const()[name = tensor("input_13_perm_0"), val = tensor([0, -1, 1])]; + tensor input_13_cast_fp16 = transpose(perm = input_13_perm_0, x = x_9_cast_fp16)[name = tensor("transpose_173")]; + tensor input_15_cast_fp16 = leaky_relu(alpha = var_20, x = input_13_cast_fp16)[name = tensor("input_15_cast_fp16")]; + tensor x_11_pad_type_0 = const()[name = tensor("x_11_pad_type_0"), val = tensor("custom")]; + tensor x_11_pad_0 = const()[name = tensor("x_11_pad_0"), val = tensor([2, 2])]; + tensor x_11_strides_0 = const()[name = tensor("x_11_strides_0"), val = tensor([1])]; + tensor x_11_dilations_0 = const()[name = tensor("x_11_dilations_0"), val = tensor([1])]; + tensor x_11_groups_0 = const()[name = tensor("x_11_groups_0"), val = tensor(1)]; + tensor weight_11_to_fp16 = const()[name = tensor("weight_11_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(5431936)))]; + tensor text_encoder_cnn_2_0_bias_to_fp16 = const()[name = tensor("text_encoder_cnn_2_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(8053440)))]; + tensor x_11_cast_fp16 = conv(bias = text_encoder_cnn_2_0_bias_to_fp16, dilations = x_11_dilations_0, groups = x_11_groups_0, pad = x_11_pad_0, pad_type = x_11_pad_type_0, strides = x_11_strides_0, weight = weight_11_to_fp16, x = input_15_cast_fp16)[name = tensor("x_11_cast_fp16")]; + tensor input_19_perm_0 = const()[name = tensor("input_19_perm_0"), val = tensor([0, -1, 1])]; + tensor x_13_axes_0 = const()[name = tensor("x_13_axes_0"), val = tensor([-1])]; + tensor text_encoder_cnn_2_1_gamma_to_fp16 = const()[name = tensor("text_encoder_cnn_2_1_gamma_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(8054528)))]; + tensor text_encoder_cnn_2_1_beta_to_fp16 = const()[name = tensor("text_encoder_cnn_2_1_beta_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(8055616)))]; + tensor input_19_cast_fp16 = transpose(perm = input_19_perm_0, x = x_11_cast_fp16)[name = tensor("transpose_172")]; + tensor x_13_cast_fp16 = layer_norm(axes = x_13_axes_0, beta = text_encoder_cnn_2_1_beta_to_fp16, epsilon = var_18_to_fp16, gamma = text_encoder_cnn_2_1_gamma_to_fp16, x = input_19_cast_fp16)[name = tensor("x_13_cast_fp16")]; + tensor input_23_cast_fp16 = leaky_relu(alpha = var_20, x = x_13_cast_fp16)[name = tensor("input_23_cast_fp16")]; + tensor input_25_batch_first_transpose_perm_0 = const()[name = tensor("input_25_batch_first_transpose_perm_0"), val = tensor([1, 0, 2])]; + tensor x_17_batch_first_direction_0 = const()[name = tensor("x_17_batch_first_direction_0"), val = tensor("bidirectional")]; + tensor x_17_batch_first_output_sequence_0 = const()[name = tensor("x_17_batch_first_output_sequence_0"), val = tensor(true)]; + tensor x_17_batch_first_recurrent_activation_0 = const()[name = tensor("x_17_batch_first_recurrent_activation_0"), val = tensor("sigmoid")]; + tensor x_17_batch_first_cell_activation_0 = const()[name = tensor("x_17_batch_first_cell_activation_0"), val = tensor("tanh")]; + tensor x_17_batch_first_activation_0 = const()[name = tensor("x_17_batch_first_activation_0"), val = tensor("tanh")]; + tensor x_17_batch_first_lstm_h0_reshaped_to_fp16 = const()[name = tensor("x_17_batch_first_lstm_h0_reshaped_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(8056704)))]; + tensor concat_4_to_fp16 = const()[name = tensor("concat_4_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(8057792)))]; + tensor concat_5_to_fp16 = const()[name = tensor("concat_5_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(9106432)))]; + tensor add_0_to_fp16 = const()[name = tensor("add_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(9630784)))]; + tensor concat_6_to_fp16 = const()[name = tensor("concat_6_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(9632896)))]; + tensor concat_7_to_fp16 = const()[name = tensor("concat_7_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(10681536)))]; + tensor add_1_to_fp16 = const()[name = tensor("add_1_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11205888)))]; + tensor input_25_batch_first_transpose_cast_fp16 = transpose(perm = input_25_batch_first_transpose_perm_0, x = input_23_cast_fp16)[name = tensor("transpose_171")]; + tensor x_17_batch_first_cast_fp16_0, tensor x_17_batch_first_cast_fp16_1, tensor x_17_batch_first_cast_fp16_2 = lstm(activation = x_17_batch_first_activation_0, bias = add_0_to_fp16, bias_back = add_1_to_fp16, cell_activation = x_17_batch_first_cell_activation_0, direction = x_17_batch_first_direction_0, initial_c = x_17_batch_first_lstm_h0_reshaped_to_fp16, initial_h = x_17_batch_first_lstm_h0_reshaped_to_fp16, output_sequence = x_17_batch_first_output_sequence_0, recurrent_activation = x_17_batch_first_recurrent_activation_0, weight_hh = concat_5_to_fp16, weight_hh_back = concat_7_to_fp16, weight_ih = concat_4_to_fp16, weight_ih_back = concat_6_to_fp16, x = input_25_batch_first_transpose_cast_fp16)[name = tensor("x_17_batch_first_cast_fp16")]; + tensor transpose_30_perm_0 = const()[name = tensor("transpose_30_perm_0"), val = tensor([1, 2, 0])]; + tensor inputs_embeds_axis_0 = const()[name = tensor("inputs_embeds_axis_0"), val = tensor(0)]; + tensor inputs_embeds_batch_dims_0 = const()[name = tensor("inputs_embeds_batch_dims_0"), val = tensor(0)]; + tensor inputs_embeds_validate_indices_0 = const()[name = tensor("inputs_embeds_validate_indices_0"), val = tensor(false)]; + tensor bert_embeddings_word_embeddings_weight_to_fp16 = const()[name = tensor("bert_embeddings_word_embeddings_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11208000)))]; + tensor tokens_to_uint16_dtype_0 = const()[name = tensor("tokens_to_uint16_dtype_0"), val = tensor("uint16")]; + tensor tokens_to_uint16 = cast(dtype = tokens_to_uint16_dtype_0, x = tokens)[name = tensor("cast_63")]; + tensor inputs_embeds_cast_fp16_cast_uint16 = gather(axis = inputs_embeds_axis_0, batch_dims = inputs_embeds_batch_dims_0, indices = tokens_to_uint16, validate_indices = inputs_embeds_validate_indices_0, x = bert_embeddings_word_embeddings_weight_to_fp16)[name = tensor("inputs_embeds_cast_fp16_cast_uint16")]; + tensor token_type_embeddings_1_to_fp16 = const()[name = tensor("token_type_embeddings_1_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11253632)))]; + tensor embeddings_1_cast_fp16 = add(x = inputs_embeds_cast_fp16_cast_uint16, y = token_type_embeddings_1_to_fp16)[name = tensor("embeddings_1_cast_fp16")]; + tensor position_embeddings_1_to_fp16 = const()[name = tensor("position_embeddings_1_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11319232)))]; + tensor input_31_cast_fp16 = add(x = embeddings_1_cast_fp16, y = position_embeddings_1_to_fp16)[name = tensor("input_31_cast_fp16")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor bert_embeddings_LayerNorm_weight_to_fp16 = const()[name = tensor("bert_embeddings_LayerNorm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11384832)))]; + tensor bert_embeddings_LayerNorm_bias_to_fp16 = const()[name = tensor("bert_embeddings_LayerNorm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11385152)))]; + tensor var_131_to_fp16 = const()[name = tensor("op_131_to_fp16"), val = tensor(0x1p-24)]; + tensor input_33_cast_fp16 = layer_norm(axes = input_33_axes_0, beta = bert_embeddings_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_embeddings_LayerNorm_weight_to_fp16, x = input_31_cast_fp16)[name = tensor("input_33_cast_fp16")]; + tensor bert_encoder_embedding_hidden_mapping_in_weight_to_fp16 = const()[name = tensor("bert_encoder_embedding_hidden_mapping_in_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11385472)))]; + tensor bert_encoder_embedding_hidden_mapping_in_bias_to_fp16 = const()[name = tensor("bert_encoder_embedding_hidden_mapping_in_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11582144)))]; + tensor linear_0_cast_fp16 = linear(bias = bert_encoder_embedding_hidden_mapping_in_bias_to_fp16, weight = bert_encoder_embedding_hidden_mapping_in_weight_to_fp16, x = input_33_cast_fp16)[name = tensor("linear_0_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(11583744)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(12763456)))]; + tensor linear_1_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = linear_0_cast_fp16)[name = tensor("linear_1_cast_fp16")]; + tensor var_213 = const()[name = tensor("op_213"), val = tensor([1, -1, 12, 64])]; + tensor var_214_cast_fp16 = reshape(shape = var_213, x = linear_1_cast_fp16)[name = tensor("op_214_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(12765056)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(13944768)))]; + tensor linear_2_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = linear_0_cast_fp16)[name = tensor("linear_2_cast_fp16")]; + tensor var_219 = const()[name = tensor("op_219"), val = tensor([1, -1, 12, 64])]; + tensor var_220_cast_fp16 = reshape(shape = var_219, x = linear_2_cast_fp16)[name = tensor("op_220_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(13946368)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(15126080)))]; + tensor linear_3_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = linear_0_cast_fp16)[name = tensor("linear_3_cast_fp16")]; + tensor var_225 = const()[name = tensor("op_225"), val = tensor([1, -1, 12, 64])]; + tensor var_226_cast_fp16 = reshape(shape = var_225, x = linear_3_cast_fp16)[name = tensor("op_226_cast_fp16")]; + tensor value_layer_1_perm_0 = const()[name = tensor("value_layer_1_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_0_y_0_to_fp16 = const()[name = tensor("mul_0_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_0_cast_fp16 = mul(x = var_214_cast_fp16, y = mul_0_y_0_to_fp16)[name = tensor("mul_0_cast_fp16")]; + tensor matmul_0_transpose_y_0 = const()[name = tensor("matmul_0_transpose_y_0"), val = tensor(true)]; + tensor matmul_0_transpose_x_0 = const()[name = tensor("matmul_0_transpose_x_0"), val = tensor(false)]; + tensor transpose_72_perm_0 = const()[name = tensor("transpose_72_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_73_perm_0 = const()[name = tensor("transpose_73_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_73 = transpose(perm = transpose_73_perm_0, x = var_220_cast_fp16)[name = tensor("transpose_167")]; + tensor transpose_72 = transpose(perm = transpose_72_perm_0, x = mul_0_cast_fp16)[name = tensor("transpose_168")]; + tensor matmul_0_cast_fp16 = matmul(transpose_x = matmul_0_transpose_x_0, transpose_y = matmul_0_transpose_y_0, x = transpose_72, y = transpose_73)[name = tensor("matmul_0_cast_fp16")]; + tensor softmax_0_axis_0 = const()[name = tensor("softmax_0_axis_0"), val = tensor(-1)]; + tensor softmax_0_cast_fp16 = softmax(axis = softmax_0_axis_0, x = matmul_0_cast_fp16)[name = tensor("softmax_0_cast_fp16")]; + tensor attention_output_1_transpose_x_0 = const()[name = tensor("attention_output_1_transpose_x_0"), val = tensor(false)]; + tensor attention_output_1_transpose_y_0 = const()[name = tensor("attention_output_1_transpose_y_0"), val = tensor(false)]; + tensor value_layer_1_cast_fp16 = transpose(perm = value_layer_1_perm_0, x = var_226_cast_fp16)[name = tensor("transpose_169")]; + tensor attention_output_1_cast_fp16 = matmul(transpose_x = attention_output_1_transpose_x_0, transpose_y = attention_output_1_transpose_y_0, x = softmax_0_cast_fp16, y = value_layer_1_cast_fp16)[name = tensor("attention_output_1_cast_fp16")]; + tensor attention_output_3_perm_0 = const()[name = tensor("attention_output_3_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_230 = const()[name = tensor("op_230"), val = tensor([1, 256, 768])]; + tensor attention_output_3_cast_fp16 = transpose(perm = attention_output_3_perm_0, x = attention_output_1_cast_fp16)[name = tensor("transpose_166")]; + tensor input_37_cast_fp16 = reshape(shape = var_230, x = attention_output_3_cast_fp16)[name = tensor("input_37_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(15127680)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(16307392)))]; + tensor linear_4_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_37_cast_fp16)[name = tensor("linear_4_cast_fp16")]; + tensor input_41_cast_fp16 = add(x = linear_0_cast_fp16, y = linear_4_cast_fp16)[name = tensor("input_41_cast_fp16")]; + tensor input_43_axes_0 = const()[name = tensor("input_43_axes_0"), val = tensor([-1])]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(16308992)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(16310592)))]; + tensor input_43_cast_fp16 = layer_norm(axes = input_43_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_41_cast_fp16)[name = tensor("input_43_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(16312192)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(19457984)))]; + tensor linear_5_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_43_cast_fp16)[name = tensor("linear_5_cast_fp16")]; + tensor input_47_mode_0 = const()[name = tensor("input_47_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_47_cast_fp16 = gelu(mode = input_47_mode_0, x = linear_5_cast_fp16)[name = tensor("input_47_cast_fp16")]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(19462144)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(22607936)))]; + tensor linear_6_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_47_cast_fp16)[name = tensor("linear_6_cast_fp16")]; + tensor input_49_cast_fp16 = add(x = linear_6_cast_fp16, y = input_43_cast_fp16)[name = tensor("input_49_cast_fp16")]; + tensor hidden_states_3_axes_0 = const()[name = tensor("hidden_states_3_axes_0"), val = tensor([-1])]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(22609536)))]; + tensor bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16 = const()[name = tensor("bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(22611136)))]; + tensor hidden_states_3_cast_fp16 = layer_norm(axes = hidden_states_3_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_49_cast_fp16)[name = tensor("hidden_states_3_cast_fp16")]; + tensor linear_7_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = tensor("linear_7_cast_fp16")]; + tensor var_280 = const()[name = tensor("op_280"), val = tensor([1, -1, 12, 64])]; + tensor var_281_cast_fp16 = reshape(shape = var_280, x = linear_7_cast_fp16)[name = tensor("op_281_cast_fp16")]; + tensor linear_8_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = tensor("linear_8_cast_fp16")]; + tensor var_286 = const()[name = tensor("op_286"), val = tensor([1, -1, 12, 64])]; + tensor var_287_cast_fp16 = reshape(shape = var_286, x = linear_8_cast_fp16)[name = tensor("op_287_cast_fp16")]; + tensor linear_9_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = tensor("linear_9_cast_fp16")]; + tensor var_292 = const()[name = tensor("op_292"), val = tensor([1, -1, 12, 64])]; + tensor var_293_cast_fp16 = reshape(shape = var_292, x = linear_9_cast_fp16)[name = tensor("op_293_cast_fp16")]; + tensor value_layer_3_perm_0 = const()[name = tensor("value_layer_3_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_1_y_0_to_fp16 = const()[name = tensor("mul_1_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_1_cast_fp16 = mul(x = var_281_cast_fp16, y = mul_1_y_0_to_fp16)[name = tensor("mul_1_cast_fp16")]; + tensor matmul_1_transpose_y_0 = const()[name = tensor("matmul_1_transpose_y_0"), val = tensor(true)]; + tensor matmul_1_transpose_x_0 = const()[name = tensor("matmul_1_transpose_x_0"), val = tensor(false)]; + tensor transpose_74_perm_0 = const()[name = tensor("transpose_74_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_75_perm_0 = const()[name = tensor("transpose_75_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_75 = transpose(perm = transpose_75_perm_0, x = var_287_cast_fp16)[name = tensor("transpose_163")]; + tensor transpose_74 = transpose(perm = transpose_74_perm_0, x = mul_1_cast_fp16)[name = tensor("transpose_164")]; + tensor matmul_1_cast_fp16 = matmul(transpose_x = matmul_1_transpose_x_0, transpose_y = matmul_1_transpose_y_0, x = transpose_74, y = transpose_75)[name = tensor("matmul_1_cast_fp16")]; + tensor softmax_1_axis_0 = const()[name = tensor("softmax_1_axis_0"), val = tensor(-1)]; + tensor softmax_1_cast_fp16 = softmax(axis = softmax_1_axis_0, x = matmul_1_cast_fp16)[name = tensor("softmax_1_cast_fp16")]; + tensor attention_output_5_transpose_x_0 = const()[name = tensor("attention_output_5_transpose_x_0"), val = tensor(false)]; + tensor attention_output_5_transpose_y_0 = const()[name = tensor("attention_output_5_transpose_y_0"), val = tensor(false)]; + tensor value_layer_3_cast_fp16 = transpose(perm = value_layer_3_perm_0, x = var_293_cast_fp16)[name = tensor("transpose_165")]; + tensor attention_output_5_cast_fp16 = matmul(transpose_x = attention_output_5_transpose_x_0, transpose_y = attention_output_5_transpose_y_0, x = softmax_1_cast_fp16, y = value_layer_3_cast_fp16)[name = tensor("attention_output_5_cast_fp16")]; + tensor attention_output_7_perm_0 = const()[name = tensor("attention_output_7_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_297 = const()[name = tensor("op_297"), val = tensor([1, 256, 768])]; + tensor attention_output_7_cast_fp16 = transpose(perm = attention_output_7_perm_0, x = attention_output_5_cast_fp16)[name = tensor("transpose_162")]; + tensor input_51_cast_fp16 = reshape(shape = var_297, x = attention_output_7_cast_fp16)[name = tensor("input_51_cast_fp16")]; + tensor linear_10_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_51_cast_fp16)[name = tensor("linear_10_cast_fp16")]; + tensor input_55_cast_fp16 = add(x = hidden_states_3_cast_fp16, y = linear_10_cast_fp16)[name = tensor("input_55_cast_fp16")]; + tensor input_57_axes_0 = const()[name = tensor("input_57_axes_0"), val = tensor([-1])]; + tensor input_57_cast_fp16 = layer_norm(axes = input_57_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_55_cast_fp16)[name = tensor("input_57_cast_fp16")]; + tensor linear_11_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_57_cast_fp16)[name = tensor("linear_11_cast_fp16")]; + tensor input_61_mode_0 = const()[name = tensor("input_61_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_61_cast_fp16 = gelu(mode = input_61_mode_0, x = linear_11_cast_fp16)[name = tensor("input_61_cast_fp16")]; + tensor linear_12_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_61_cast_fp16)[name = tensor("linear_12_cast_fp16")]; + tensor input_63_cast_fp16 = add(x = linear_12_cast_fp16, y = input_57_cast_fp16)[name = tensor("input_63_cast_fp16")]; + tensor hidden_states_5_axes_0 = const()[name = tensor("hidden_states_5_axes_0"), val = tensor([-1])]; + tensor hidden_states_5_cast_fp16 = layer_norm(axes = hidden_states_5_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_63_cast_fp16)[name = tensor("hidden_states_5_cast_fp16")]; + tensor linear_13_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = tensor("linear_13_cast_fp16")]; + tensor var_347 = const()[name = tensor("op_347"), val = tensor([1, -1, 12, 64])]; + tensor var_348_cast_fp16 = reshape(shape = var_347, x = linear_13_cast_fp16)[name = tensor("op_348_cast_fp16")]; + tensor linear_14_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = tensor("linear_14_cast_fp16")]; + tensor var_353 = const()[name = tensor("op_353"), val = tensor([1, -1, 12, 64])]; + tensor var_354_cast_fp16 = reshape(shape = var_353, x = linear_14_cast_fp16)[name = tensor("op_354_cast_fp16")]; + tensor linear_15_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = tensor("linear_15_cast_fp16")]; + tensor var_359 = const()[name = tensor("op_359"), val = tensor([1, -1, 12, 64])]; + tensor var_360_cast_fp16 = reshape(shape = var_359, x = linear_15_cast_fp16)[name = tensor("op_360_cast_fp16")]; + tensor value_layer_5_perm_0 = const()[name = tensor("value_layer_5_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_2_y_0_to_fp16 = const()[name = tensor("mul_2_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_2_cast_fp16 = mul(x = var_348_cast_fp16, y = mul_2_y_0_to_fp16)[name = tensor("mul_2_cast_fp16")]; + tensor matmul_2_transpose_y_0 = const()[name = tensor("matmul_2_transpose_y_0"), val = tensor(true)]; + tensor matmul_2_transpose_x_0 = const()[name = tensor("matmul_2_transpose_x_0"), val = tensor(false)]; + tensor transpose_76_perm_0 = const()[name = tensor("transpose_76_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_77_perm_0 = const()[name = tensor("transpose_77_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_77 = transpose(perm = transpose_77_perm_0, x = var_354_cast_fp16)[name = tensor("transpose_159")]; + tensor transpose_76 = transpose(perm = transpose_76_perm_0, x = mul_2_cast_fp16)[name = tensor("transpose_160")]; + tensor matmul_2_cast_fp16 = matmul(transpose_x = matmul_2_transpose_x_0, transpose_y = matmul_2_transpose_y_0, x = transpose_76, y = transpose_77)[name = tensor("matmul_2_cast_fp16")]; + tensor softmax_2_axis_0 = const()[name = tensor("softmax_2_axis_0"), val = tensor(-1)]; + tensor softmax_2_cast_fp16 = softmax(axis = softmax_2_axis_0, x = matmul_2_cast_fp16)[name = tensor("softmax_2_cast_fp16")]; + tensor attention_output_9_transpose_x_0 = const()[name = tensor("attention_output_9_transpose_x_0"), val = tensor(false)]; + tensor attention_output_9_transpose_y_0 = const()[name = tensor("attention_output_9_transpose_y_0"), val = tensor(false)]; + tensor value_layer_5_cast_fp16 = transpose(perm = value_layer_5_perm_0, x = var_360_cast_fp16)[name = tensor("transpose_161")]; + tensor attention_output_9_cast_fp16 = matmul(transpose_x = attention_output_9_transpose_x_0, transpose_y = attention_output_9_transpose_y_0, x = softmax_2_cast_fp16, y = value_layer_5_cast_fp16)[name = tensor("attention_output_9_cast_fp16")]; + tensor attention_output_11_perm_0 = const()[name = tensor("attention_output_11_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_364 = const()[name = tensor("op_364"), val = tensor([1, 256, 768])]; + tensor attention_output_11_cast_fp16 = transpose(perm = attention_output_11_perm_0, x = attention_output_9_cast_fp16)[name = tensor("transpose_158")]; + tensor input_65_cast_fp16 = reshape(shape = var_364, x = attention_output_11_cast_fp16)[name = tensor("input_65_cast_fp16")]; + tensor linear_16_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_65_cast_fp16)[name = tensor("linear_16_cast_fp16")]; + tensor input_69_cast_fp16 = add(x = hidden_states_5_cast_fp16, y = linear_16_cast_fp16)[name = tensor("input_69_cast_fp16")]; + tensor input_71_axes_0 = const()[name = tensor("input_71_axes_0"), val = tensor([-1])]; + tensor input_71_cast_fp16 = layer_norm(axes = input_71_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_69_cast_fp16)[name = tensor("input_71_cast_fp16")]; + tensor linear_17_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_71_cast_fp16)[name = tensor("linear_17_cast_fp16")]; + tensor input_75_mode_0 = const()[name = tensor("input_75_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_75_cast_fp16 = gelu(mode = input_75_mode_0, x = linear_17_cast_fp16)[name = tensor("input_75_cast_fp16")]; + tensor linear_18_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_75_cast_fp16)[name = tensor("linear_18_cast_fp16")]; + tensor input_77_cast_fp16 = add(x = linear_18_cast_fp16, y = input_71_cast_fp16)[name = tensor("input_77_cast_fp16")]; + tensor hidden_states_7_axes_0 = const()[name = tensor("hidden_states_7_axes_0"), val = tensor([-1])]; + tensor hidden_states_7_cast_fp16 = layer_norm(axes = hidden_states_7_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_77_cast_fp16)[name = tensor("hidden_states_7_cast_fp16")]; + tensor linear_19_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = tensor("linear_19_cast_fp16")]; + tensor var_414 = const()[name = tensor("op_414"), val = tensor([1, -1, 12, 64])]; + tensor var_415_cast_fp16 = reshape(shape = var_414, x = linear_19_cast_fp16)[name = tensor("op_415_cast_fp16")]; + tensor linear_20_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = tensor("linear_20_cast_fp16")]; + tensor var_420 = const()[name = tensor("op_420"), val = tensor([1, -1, 12, 64])]; + tensor var_421_cast_fp16 = reshape(shape = var_420, x = linear_20_cast_fp16)[name = tensor("op_421_cast_fp16")]; + tensor linear_21_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = tensor("linear_21_cast_fp16")]; + tensor var_426 = const()[name = tensor("op_426"), val = tensor([1, -1, 12, 64])]; + tensor var_427_cast_fp16 = reshape(shape = var_426, x = linear_21_cast_fp16)[name = tensor("op_427_cast_fp16")]; + tensor value_layer_7_perm_0 = const()[name = tensor("value_layer_7_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_3_y_0_to_fp16 = const()[name = tensor("mul_3_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_3_cast_fp16 = mul(x = var_415_cast_fp16, y = mul_3_y_0_to_fp16)[name = tensor("mul_3_cast_fp16")]; + tensor matmul_3_transpose_y_0 = const()[name = tensor("matmul_3_transpose_y_0"), val = tensor(true)]; + tensor matmul_3_transpose_x_0 = const()[name = tensor("matmul_3_transpose_x_0"), val = tensor(false)]; + tensor transpose_78_perm_0 = const()[name = tensor("transpose_78_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_79_perm_0 = const()[name = tensor("transpose_79_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_79 = transpose(perm = transpose_79_perm_0, x = var_421_cast_fp16)[name = tensor("transpose_155")]; + tensor transpose_78 = transpose(perm = transpose_78_perm_0, x = mul_3_cast_fp16)[name = tensor("transpose_156")]; + tensor matmul_3_cast_fp16 = matmul(transpose_x = matmul_3_transpose_x_0, transpose_y = matmul_3_transpose_y_0, x = transpose_78, y = transpose_79)[name = tensor("matmul_3_cast_fp16")]; + tensor softmax_3_axis_0 = const()[name = tensor("softmax_3_axis_0"), val = tensor(-1)]; + tensor softmax_3_cast_fp16 = softmax(axis = softmax_3_axis_0, x = matmul_3_cast_fp16)[name = tensor("softmax_3_cast_fp16")]; + tensor attention_output_13_transpose_x_0 = const()[name = tensor("attention_output_13_transpose_x_0"), val = tensor(false)]; + tensor attention_output_13_transpose_y_0 = const()[name = tensor("attention_output_13_transpose_y_0"), val = tensor(false)]; + tensor value_layer_7_cast_fp16 = transpose(perm = value_layer_7_perm_0, x = var_427_cast_fp16)[name = tensor("transpose_157")]; + tensor attention_output_13_cast_fp16 = matmul(transpose_x = attention_output_13_transpose_x_0, transpose_y = attention_output_13_transpose_y_0, x = softmax_3_cast_fp16, y = value_layer_7_cast_fp16)[name = tensor("attention_output_13_cast_fp16")]; + tensor attention_output_15_perm_0 = const()[name = tensor("attention_output_15_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_431 = const()[name = tensor("op_431"), val = tensor([1, 256, 768])]; + tensor attention_output_15_cast_fp16 = transpose(perm = attention_output_15_perm_0, x = attention_output_13_cast_fp16)[name = tensor("transpose_154")]; + tensor input_79_cast_fp16 = reshape(shape = var_431, x = attention_output_15_cast_fp16)[name = tensor("input_79_cast_fp16")]; + tensor linear_22_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_79_cast_fp16)[name = tensor("linear_22_cast_fp16")]; + tensor input_83_cast_fp16 = add(x = hidden_states_7_cast_fp16, y = linear_22_cast_fp16)[name = tensor("input_83_cast_fp16")]; + tensor input_85_axes_0 = const()[name = tensor("input_85_axes_0"), val = tensor([-1])]; + tensor input_85_cast_fp16 = layer_norm(axes = input_85_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_83_cast_fp16)[name = tensor("input_85_cast_fp16")]; + tensor linear_23_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_85_cast_fp16)[name = tensor("linear_23_cast_fp16")]; + tensor input_89_mode_0 = const()[name = tensor("input_89_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_89_cast_fp16 = gelu(mode = input_89_mode_0, x = linear_23_cast_fp16)[name = tensor("input_89_cast_fp16")]; + tensor linear_24_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_89_cast_fp16)[name = tensor("linear_24_cast_fp16")]; + tensor input_91_cast_fp16 = add(x = linear_24_cast_fp16, y = input_85_cast_fp16)[name = tensor("input_91_cast_fp16")]; + tensor hidden_states_9_axes_0 = const()[name = tensor("hidden_states_9_axes_0"), val = tensor([-1])]; + tensor hidden_states_9_cast_fp16 = layer_norm(axes = hidden_states_9_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_91_cast_fp16)[name = tensor("hidden_states_9_cast_fp16")]; + tensor linear_25_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = tensor("linear_25_cast_fp16")]; + tensor var_481 = const()[name = tensor("op_481"), val = tensor([1, -1, 12, 64])]; + tensor var_482_cast_fp16 = reshape(shape = var_481, x = linear_25_cast_fp16)[name = tensor("op_482_cast_fp16")]; + tensor linear_26_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = tensor("linear_26_cast_fp16")]; + tensor var_487 = const()[name = tensor("op_487"), val = tensor([1, -1, 12, 64])]; + tensor var_488_cast_fp16 = reshape(shape = var_487, x = linear_26_cast_fp16)[name = tensor("op_488_cast_fp16")]; + tensor linear_27_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = tensor("linear_27_cast_fp16")]; + tensor var_493 = const()[name = tensor("op_493"), val = tensor([1, -1, 12, 64])]; + tensor var_494_cast_fp16 = reshape(shape = var_493, x = linear_27_cast_fp16)[name = tensor("op_494_cast_fp16")]; + tensor value_layer_9_perm_0 = const()[name = tensor("value_layer_9_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_4_y_0_to_fp16 = const()[name = tensor("mul_4_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_4_cast_fp16 = mul(x = var_482_cast_fp16, y = mul_4_y_0_to_fp16)[name = tensor("mul_4_cast_fp16")]; + tensor matmul_4_transpose_y_0 = const()[name = tensor("matmul_4_transpose_y_0"), val = tensor(true)]; + tensor matmul_4_transpose_x_0 = const()[name = tensor("matmul_4_transpose_x_0"), val = tensor(false)]; + tensor transpose_80_perm_0 = const()[name = tensor("transpose_80_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_81_perm_0 = const()[name = tensor("transpose_81_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_81 = transpose(perm = transpose_81_perm_0, x = var_488_cast_fp16)[name = tensor("transpose_151")]; + tensor transpose_80 = transpose(perm = transpose_80_perm_0, x = mul_4_cast_fp16)[name = tensor("transpose_152")]; + tensor matmul_4_cast_fp16 = matmul(transpose_x = matmul_4_transpose_x_0, transpose_y = matmul_4_transpose_y_0, x = transpose_80, y = transpose_81)[name = tensor("matmul_4_cast_fp16")]; + tensor softmax_4_axis_0 = const()[name = tensor("softmax_4_axis_0"), val = tensor(-1)]; + tensor softmax_4_cast_fp16 = softmax(axis = softmax_4_axis_0, x = matmul_4_cast_fp16)[name = tensor("softmax_4_cast_fp16")]; + tensor attention_output_17_transpose_x_0 = const()[name = tensor("attention_output_17_transpose_x_0"), val = tensor(false)]; + tensor attention_output_17_transpose_y_0 = const()[name = tensor("attention_output_17_transpose_y_0"), val = tensor(false)]; + tensor value_layer_9_cast_fp16 = transpose(perm = value_layer_9_perm_0, x = var_494_cast_fp16)[name = tensor("transpose_153")]; + tensor attention_output_17_cast_fp16 = matmul(transpose_x = attention_output_17_transpose_x_0, transpose_y = attention_output_17_transpose_y_0, x = softmax_4_cast_fp16, y = value_layer_9_cast_fp16)[name = tensor("attention_output_17_cast_fp16")]; + tensor attention_output_19_perm_0 = const()[name = tensor("attention_output_19_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_498 = const()[name = tensor("op_498"), val = tensor([1, 256, 768])]; + tensor attention_output_19_cast_fp16 = transpose(perm = attention_output_19_perm_0, x = attention_output_17_cast_fp16)[name = tensor("transpose_150")]; + tensor input_93_cast_fp16 = reshape(shape = var_498, x = attention_output_19_cast_fp16)[name = tensor("input_93_cast_fp16")]; + tensor linear_28_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_93_cast_fp16)[name = tensor("linear_28_cast_fp16")]; + tensor input_97_cast_fp16 = add(x = hidden_states_9_cast_fp16, y = linear_28_cast_fp16)[name = tensor("input_97_cast_fp16")]; + tensor input_99_axes_0 = const()[name = tensor("input_99_axes_0"), val = tensor([-1])]; + tensor input_99_cast_fp16 = layer_norm(axes = input_99_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_97_cast_fp16)[name = tensor("input_99_cast_fp16")]; + tensor linear_29_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_99_cast_fp16)[name = tensor("linear_29_cast_fp16")]; + tensor input_103_mode_0 = const()[name = tensor("input_103_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_103_cast_fp16 = gelu(mode = input_103_mode_0, x = linear_29_cast_fp16)[name = tensor("input_103_cast_fp16")]; + tensor linear_30_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_103_cast_fp16)[name = tensor("linear_30_cast_fp16")]; + tensor input_105_cast_fp16 = add(x = linear_30_cast_fp16, y = input_99_cast_fp16)[name = tensor("input_105_cast_fp16")]; + tensor hidden_states_11_axes_0 = const()[name = tensor("hidden_states_11_axes_0"), val = tensor([-1])]; + tensor hidden_states_11_cast_fp16 = layer_norm(axes = hidden_states_11_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_105_cast_fp16)[name = tensor("hidden_states_11_cast_fp16")]; + tensor linear_31_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = tensor("linear_31_cast_fp16")]; + tensor var_548 = const()[name = tensor("op_548"), val = tensor([1, -1, 12, 64])]; + tensor var_549_cast_fp16 = reshape(shape = var_548, x = linear_31_cast_fp16)[name = tensor("op_549_cast_fp16")]; + tensor linear_32_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = tensor("linear_32_cast_fp16")]; + tensor var_554 = const()[name = tensor("op_554"), val = tensor([1, -1, 12, 64])]; + tensor var_555_cast_fp16 = reshape(shape = var_554, x = linear_32_cast_fp16)[name = tensor("op_555_cast_fp16")]; + tensor linear_33_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = tensor("linear_33_cast_fp16")]; + tensor var_560 = const()[name = tensor("op_560"), val = tensor([1, -1, 12, 64])]; + tensor var_561_cast_fp16 = reshape(shape = var_560, x = linear_33_cast_fp16)[name = tensor("op_561_cast_fp16")]; + tensor value_layer_11_perm_0 = const()[name = tensor("value_layer_11_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_5_y_0_to_fp16 = const()[name = tensor("mul_5_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_5_cast_fp16 = mul(x = var_549_cast_fp16, y = mul_5_y_0_to_fp16)[name = tensor("mul_5_cast_fp16")]; + tensor matmul_5_transpose_y_0 = const()[name = tensor("matmul_5_transpose_y_0"), val = tensor(true)]; + tensor matmul_5_transpose_x_0 = const()[name = tensor("matmul_5_transpose_x_0"), val = tensor(false)]; + tensor transpose_82_perm_0 = const()[name = tensor("transpose_82_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_83_perm_0 = const()[name = tensor("transpose_83_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_83 = transpose(perm = transpose_83_perm_0, x = var_555_cast_fp16)[name = tensor("transpose_147")]; + tensor transpose_82 = transpose(perm = transpose_82_perm_0, x = mul_5_cast_fp16)[name = tensor("transpose_148")]; + tensor matmul_5_cast_fp16 = matmul(transpose_x = matmul_5_transpose_x_0, transpose_y = matmul_5_transpose_y_0, x = transpose_82, y = transpose_83)[name = tensor("matmul_5_cast_fp16")]; + tensor softmax_5_axis_0 = const()[name = tensor("softmax_5_axis_0"), val = tensor(-1)]; + tensor softmax_5_cast_fp16 = softmax(axis = softmax_5_axis_0, x = matmul_5_cast_fp16)[name = tensor("softmax_5_cast_fp16")]; + tensor attention_output_21_transpose_x_0 = const()[name = tensor("attention_output_21_transpose_x_0"), val = tensor(false)]; + tensor attention_output_21_transpose_y_0 = const()[name = tensor("attention_output_21_transpose_y_0"), val = tensor(false)]; + tensor value_layer_11_cast_fp16 = transpose(perm = value_layer_11_perm_0, x = var_561_cast_fp16)[name = tensor("transpose_149")]; + tensor attention_output_21_cast_fp16 = matmul(transpose_x = attention_output_21_transpose_x_0, transpose_y = attention_output_21_transpose_y_0, x = softmax_5_cast_fp16, y = value_layer_11_cast_fp16)[name = tensor("attention_output_21_cast_fp16")]; + tensor attention_output_23_perm_0 = const()[name = tensor("attention_output_23_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_565 = const()[name = tensor("op_565"), val = tensor([1, 256, 768])]; + tensor attention_output_23_cast_fp16 = transpose(perm = attention_output_23_perm_0, x = attention_output_21_cast_fp16)[name = tensor("transpose_146")]; + tensor input_107_cast_fp16 = reshape(shape = var_565, x = attention_output_23_cast_fp16)[name = tensor("input_107_cast_fp16")]; + tensor linear_34_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_107_cast_fp16)[name = tensor("linear_34_cast_fp16")]; + tensor input_111_cast_fp16 = add(x = hidden_states_11_cast_fp16, y = linear_34_cast_fp16)[name = tensor("input_111_cast_fp16")]; + tensor input_113_axes_0 = const()[name = tensor("input_113_axes_0"), val = tensor([-1])]; + tensor input_113_cast_fp16 = layer_norm(axes = input_113_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_111_cast_fp16)[name = tensor("input_113_cast_fp16")]; + tensor linear_35_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_113_cast_fp16)[name = tensor("linear_35_cast_fp16")]; + tensor input_117_mode_0 = const()[name = tensor("input_117_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_117_cast_fp16 = gelu(mode = input_117_mode_0, x = linear_35_cast_fp16)[name = tensor("input_117_cast_fp16")]; + tensor linear_36_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_117_cast_fp16)[name = tensor("linear_36_cast_fp16")]; + tensor input_119_cast_fp16 = add(x = linear_36_cast_fp16, y = input_113_cast_fp16)[name = tensor("input_119_cast_fp16")]; + tensor hidden_states_13_axes_0 = const()[name = tensor("hidden_states_13_axes_0"), val = tensor([-1])]; + tensor hidden_states_13_cast_fp16 = layer_norm(axes = hidden_states_13_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_119_cast_fp16)[name = tensor("hidden_states_13_cast_fp16")]; + tensor linear_37_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = tensor("linear_37_cast_fp16")]; + tensor var_615 = const()[name = tensor("op_615"), val = tensor([1, -1, 12, 64])]; + tensor var_616_cast_fp16 = reshape(shape = var_615, x = linear_37_cast_fp16)[name = tensor("op_616_cast_fp16")]; + tensor linear_38_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = tensor("linear_38_cast_fp16")]; + tensor var_621 = const()[name = tensor("op_621"), val = tensor([1, -1, 12, 64])]; + tensor var_622_cast_fp16 = reshape(shape = var_621, x = linear_38_cast_fp16)[name = tensor("op_622_cast_fp16")]; + tensor linear_39_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = tensor("linear_39_cast_fp16")]; + tensor var_627 = const()[name = tensor("op_627"), val = tensor([1, -1, 12, 64])]; + tensor var_628_cast_fp16 = reshape(shape = var_627, x = linear_39_cast_fp16)[name = tensor("op_628_cast_fp16")]; + tensor value_layer_13_perm_0 = const()[name = tensor("value_layer_13_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_6_y_0_to_fp16 = const()[name = tensor("mul_6_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_6_cast_fp16 = mul(x = var_616_cast_fp16, y = mul_6_y_0_to_fp16)[name = tensor("mul_6_cast_fp16")]; + tensor matmul_6_transpose_y_0 = const()[name = tensor("matmul_6_transpose_y_0"), val = tensor(true)]; + tensor matmul_6_transpose_x_0 = const()[name = tensor("matmul_6_transpose_x_0"), val = tensor(false)]; + tensor transpose_84_perm_0 = const()[name = tensor("transpose_84_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_85_perm_0 = const()[name = tensor("transpose_85_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_85 = transpose(perm = transpose_85_perm_0, x = var_622_cast_fp16)[name = tensor("transpose_143")]; + tensor transpose_84 = transpose(perm = transpose_84_perm_0, x = mul_6_cast_fp16)[name = tensor("transpose_144")]; + tensor matmul_6_cast_fp16 = matmul(transpose_x = matmul_6_transpose_x_0, transpose_y = matmul_6_transpose_y_0, x = transpose_84, y = transpose_85)[name = tensor("matmul_6_cast_fp16")]; + tensor softmax_6_axis_0 = const()[name = tensor("softmax_6_axis_0"), val = tensor(-1)]; + tensor softmax_6_cast_fp16 = softmax(axis = softmax_6_axis_0, x = matmul_6_cast_fp16)[name = tensor("softmax_6_cast_fp16")]; + tensor attention_output_25_transpose_x_0 = const()[name = tensor("attention_output_25_transpose_x_0"), val = tensor(false)]; + tensor attention_output_25_transpose_y_0 = const()[name = tensor("attention_output_25_transpose_y_0"), val = tensor(false)]; + tensor value_layer_13_cast_fp16 = transpose(perm = value_layer_13_perm_0, x = var_628_cast_fp16)[name = tensor("transpose_145")]; + tensor attention_output_25_cast_fp16 = matmul(transpose_x = attention_output_25_transpose_x_0, transpose_y = attention_output_25_transpose_y_0, x = softmax_6_cast_fp16, y = value_layer_13_cast_fp16)[name = tensor("attention_output_25_cast_fp16")]; + tensor attention_output_27_perm_0 = const()[name = tensor("attention_output_27_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_632 = const()[name = tensor("op_632"), val = tensor([1, 256, 768])]; + tensor attention_output_27_cast_fp16 = transpose(perm = attention_output_27_perm_0, x = attention_output_25_cast_fp16)[name = tensor("transpose_142")]; + tensor input_121_cast_fp16 = reshape(shape = var_632, x = attention_output_27_cast_fp16)[name = tensor("input_121_cast_fp16")]; + tensor linear_40_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_121_cast_fp16)[name = tensor("linear_40_cast_fp16")]; + tensor input_125_cast_fp16 = add(x = hidden_states_13_cast_fp16, y = linear_40_cast_fp16)[name = tensor("input_125_cast_fp16")]; + tensor input_127_axes_0 = const()[name = tensor("input_127_axes_0"), val = tensor([-1])]; + tensor input_127_cast_fp16 = layer_norm(axes = input_127_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_125_cast_fp16)[name = tensor("input_127_cast_fp16")]; + tensor linear_41_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_127_cast_fp16)[name = tensor("linear_41_cast_fp16")]; + tensor input_131_mode_0 = const()[name = tensor("input_131_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_131_cast_fp16 = gelu(mode = input_131_mode_0, x = linear_41_cast_fp16)[name = tensor("input_131_cast_fp16")]; + tensor linear_42_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_131_cast_fp16)[name = tensor("linear_42_cast_fp16")]; + tensor input_133_cast_fp16 = add(x = linear_42_cast_fp16, y = input_127_cast_fp16)[name = tensor("input_133_cast_fp16")]; + tensor hidden_states_15_axes_0 = const()[name = tensor("hidden_states_15_axes_0"), val = tensor([-1])]; + tensor hidden_states_15_cast_fp16 = layer_norm(axes = hidden_states_15_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_133_cast_fp16)[name = tensor("hidden_states_15_cast_fp16")]; + tensor linear_43_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = tensor("linear_43_cast_fp16")]; + tensor var_682 = const()[name = tensor("op_682"), val = tensor([1, -1, 12, 64])]; + tensor var_683_cast_fp16 = reshape(shape = var_682, x = linear_43_cast_fp16)[name = tensor("op_683_cast_fp16")]; + tensor linear_44_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = tensor("linear_44_cast_fp16")]; + tensor var_688 = const()[name = tensor("op_688"), val = tensor([1, -1, 12, 64])]; + tensor var_689_cast_fp16 = reshape(shape = var_688, x = linear_44_cast_fp16)[name = tensor("op_689_cast_fp16")]; + tensor linear_45_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = tensor("linear_45_cast_fp16")]; + tensor var_694 = const()[name = tensor("op_694"), val = tensor([1, -1, 12, 64])]; + tensor var_695_cast_fp16 = reshape(shape = var_694, x = linear_45_cast_fp16)[name = tensor("op_695_cast_fp16")]; + tensor value_layer_15_perm_0 = const()[name = tensor("value_layer_15_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_7_y_0_to_fp16 = const()[name = tensor("mul_7_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_7_cast_fp16 = mul(x = var_683_cast_fp16, y = mul_7_y_0_to_fp16)[name = tensor("mul_7_cast_fp16")]; + tensor matmul_7_transpose_y_0 = const()[name = tensor("matmul_7_transpose_y_0"), val = tensor(true)]; + tensor matmul_7_transpose_x_0 = const()[name = tensor("matmul_7_transpose_x_0"), val = tensor(false)]; + tensor transpose_86_perm_0 = const()[name = tensor("transpose_86_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_87_perm_0 = const()[name = tensor("transpose_87_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_87 = transpose(perm = transpose_87_perm_0, x = var_689_cast_fp16)[name = tensor("transpose_139")]; + tensor transpose_86 = transpose(perm = transpose_86_perm_0, x = mul_7_cast_fp16)[name = tensor("transpose_140")]; + tensor matmul_7_cast_fp16 = matmul(transpose_x = matmul_7_transpose_x_0, transpose_y = matmul_7_transpose_y_0, x = transpose_86, y = transpose_87)[name = tensor("matmul_7_cast_fp16")]; + tensor softmax_7_axis_0 = const()[name = tensor("softmax_7_axis_0"), val = tensor(-1)]; + tensor softmax_7_cast_fp16 = softmax(axis = softmax_7_axis_0, x = matmul_7_cast_fp16)[name = tensor("softmax_7_cast_fp16")]; + tensor attention_output_29_transpose_x_0 = const()[name = tensor("attention_output_29_transpose_x_0"), val = tensor(false)]; + tensor attention_output_29_transpose_y_0 = const()[name = tensor("attention_output_29_transpose_y_0"), val = tensor(false)]; + tensor value_layer_15_cast_fp16 = transpose(perm = value_layer_15_perm_0, x = var_695_cast_fp16)[name = tensor("transpose_141")]; + tensor attention_output_29_cast_fp16 = matmul(transpose_x = attention_output_29_transpose_x_0, transpose_y = attention_output_29_transpose_y_0, x = softmax_7_cast_fp16, y = value_layer_15_cast_fp16)[name = tensor("attention_output_29_cast_fp16")]; + tensor attention_output_31_perm_0 = const()[name = tensor("attention_output_31_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_699 = const()[name = tensor("op_699"), val = tensor([1, 256, 768])]; + tensor attention_output_31_cast_fp16 = transpose(perm = attention_output_31_perm_0, x = attention_output_29_cast_fp16)[name = tensor("transpose_138")]; + tensor input_135_cast_fp16 = reshape(shape = var_699, x = attention_output_31_cast_fp16)[name = tensor("input_135_cast_fp16")]; + tensor linear_46_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_135_cast_fp16)[name = tensor("linear_46_cast_fp16")]; + tensor input_139_cast_fp16 = add(x = hidden_states_15_cast_fp16, y = linear_46_cast_fp16)[name = tensor("input_139_cast_fp16")]; + tensor input_141_axes_0 = const()[name = tensor("input_141_axes_0"), val = tensor([-1])]; + tensor input_141_cast_fp16 = layer_norm(axes = input_141_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_139_cast_fp16)[name = tensor("input_141_cast_fp16")]; + tensor linear_47_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_141_cast_fp16)[name = tensor("linear_47_cast_fp16")]; + tensor input_145_mode_0 = const()[name = tensor("input_145_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_145_cast_fp16 = gelu(mode = input_145_mode_0, x = linear_47_cast_fp16)[name = tensor("input_145_cast_fp16")]; + tensor linear_48_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_145_cast_fp16)[name = tensor("linear_48_cast_fp16")]; + tensor input_147_cast_fp16 = add(x = linear_48_cast_fp16, y = input_141_cast_fp16)[name = tensor("input_147_cast_fp16")]; + tensor hidden_states_17_axes_0 = const()[name = tensor("hidden_states_17_axes_0"), val = tensor([-1])]; + tensor hidden_states_17_cast_fp16 = layer_norm(axes = hidden_states_17_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_147_cast_fp16)[name = tensor("hidden_states_17_cast_fp16")]; + tensor linear_49_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_17_cast_fp16)[name = tensor("linear_49_cast_fp16")]; + tensor var_749 = const()[name = tensor("op_749"), val = tensor([1, -1, 12, 64])]; + tensor var_750_cast_fp16 = reshape(shape = var_749, x = linear_49_cast_fp16)[name = tensor("op_750_cast_fp16")]; + tensor linear_50_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_17_cast_fp16)[name = tensor("linear_50_cast_fp16")]; + tensor var_755 = const()[name = tensor("op_755"), val = tensor([1, -1, 12, 64])]; + tensor var_756_cast_fp16 = reshape(shape = var_755, x = linear_50_cast_fp16)[name = tensor("op_756_cast_fp16")]; + tensor linear_51_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_17_cast_fp16)[name = tensor("linear_51_cast_fp16")]; + tensor var_761 = const()[name = tensor("op_761"), val = tensor([1, -1, 12, 64])]; + tensor var_762_cast_fp16 = reshape(shape = var_761, x = linear_51_cast_fp16)[name = tensor("op_762_cast_fp16")]; + tensor value_layer_17_perm_0 = const()[name = tensor("value_layer_17_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_8_y_0_to_fp16 = const()[name = tensor("mul_8_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_8_cast_fp16 = mul(x = var_750_cast_fp16, y = mul_8_y_0_to_fp16)[name = tensor("mul_8_cast_fp16")]; + tensor matmul_8_transpose_y_0 = const()[name = tensor("matmul_8_transpose_y_0"), val = tensor(true)]; + tensor matmul_8_transpose_x_0 = const()[name = tensor("matmul_8_transpose_x_0"), val = tensor(false)]; + tensor transpose_88_perm_0 = const()[name = tensor("transpose_88_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_89_perm_0 = const()[name = tensor("transpose_89_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_89 = transpose(perm = transpose_89_perm_0, x = var_756_cast_fp16)[name = tensor("transpose_135")]; + tensor transpose_88 = transpose(perm = transpose_88_perm_0, x = mul_8_cast_fp16)[name = tensor("transpose_136")]; + tensor matmul_8_cast_fp16 = matmul(transpose_x = matmul_8_transpose_x_0, transpose_y = matmul_8_transpose_y_0, x = transpose_88, y = transpose_89)[name = tensor("matmul_8_cast_fp16")]; + tensor softmax_8_axis_0 = const()[name = tensor("softmax_8_axis_0"), val = tensor(-1)]; + tensor softmax_8_cast_fp16 = softmax(axis = softmax_8_axis_0, x = matmul_8_cast_fp16)[name = tensor("softmax_8_cast_fp16")]; + tensor attention_output_33_transpose_x_0 = const()[name = tensor("attention_output_33_transpose_x_0"), val = tensor(false)]; + tensor attention_output_33_transpose_y_0 = const()[name = tensor("attention_output_33_transpose_y_0"), val = tensor(false)]; + tensor value_layer_17_cast_fp16 = transpose(perm = value_layer_17_perm_0, x = var_762_cast_fp16)[name = tensor("transpose_137")]; + tensor attention_output_33_cast_fp16 = matmul(transpose_x = attention_output_33_transpose_x_0, transpose_y = attention_output_33_transpose_y_0, x = softmax_8_cast_fp16, y = value_layer_17_cast_fp16)[name = tensor("attention_output_33_cast_fp16")]; + tensor attention_output_35_perm_0 = const()[name = tensor("attention_output_35_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_766 = const()[name = tensor("op_766"), val = tensor([1, 256, 768])]; + tensor attention_output_35_cast_fp16 = transpose(perm = attention_output_35_perm_0, x = attention_output_33_cast_fp16)[name = tensor("transpose_134")]; + tensor input_149_cast_fp16 = reshape(shape = var_766, x = attention_output_35_cast_fp16)[name = tensor("input_149_cast_fp16")]; + tensor linear_52_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_149_cast_fp16)[name = tensor("linear_52_cast_fp16")]; + tensor input_153_cast_fp16 = add(x = hidden_states_17_cast_fp16, y = linear_52_cast_fp16)[name = tensor("input_153_cast_fp16")]; + tensor input_155_axes_0 = const()[name = tensor("input_155_axes_0"), val = tensor([-1])]; + tensor input_155_cast_fp16 = layer_norm(axes = input_155_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_153_cast_fp16)[name = tensor("input_155_cast_fp16")]; + tensor linear_53_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_155_cast_fp16)[name = tensor("linear_53_cast_fp16")]; + tensor input_159_mode_0 = const()[name = tensor("input_159_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_159_cast_fp16 = gelu(mode = input_159_mode_0, x = linear_53_cast_fp16)[name = tensor("input_159_cast_fp16")]; + tensor linear_54_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_159_cast_fp16)[name = tensor("linear_54_cast_fp16")]; + tensor input_161_cast_fp16 = add(x = linear_54_cast_fp16, y = input_155_cast_fp16)[name = tensor("input_161_cast_fp16")]; + tensor hidden_states_19_axes_0 = const()[name = tensor("hidden_states_19_axes_0"), val = tensor([-1])]; + tensor hidden_states_19_cast_fp16 = layer_norm(axes = hidden_states_19_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_161_cast_fp16)[name = tensor("hidden_states_19_cast_fp16")]; + tensor linear_55_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_19_cast_fp16)[name = tensor("linear_55_cast_fp16")]; + tensor var_816 = const()[name = tensor("op_816"), val = tensor([1, -1, 12, 64])]; + tensor var_817_cast_fp16 = reshape(shape = var_816, x = linear_55_cast_fp16)[name = tensor("op_817_cast_fp16")]; + tensor linear_56_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_19_cast_fp16)[name = tensor("linear_56_cast_fp16")]; + tensor var_822 = const()[name = tensor("op_822"), val = tensor([1, -1, 12, 64])]; + tensor var_823_cast_fp16 = reshape(shape = var_822, x = linear_56_cast_fp16)[name = tensor("op_823_cast_fp16")]; + tensor linear_57_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_19_cast_fp16)[name = tensor("linear_57_cast_fp16")]; + tensor var_828 = const()[name = tensor("op_828"), val = tensor([1, -1, 12, 64])]; + tensor var_829_cast_fp16 = reshape(shape = var_828, x = linear_57_cast_fp16)[name = tensor("op_829_cast_fp16")]; + tensor value_layer_19_perm_0 = const()[name = tensor("value_layer_19_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_9_y_0_to_fp16 = const()[name = tensor("mul_9_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_9_cast_fp16 = mul(x = var_817_cast_fp16, y = mul_9_y_0_to_fp16)[name = tensor("mul_9_cast_fp16")]; + tensor matmul_9_transpose_y_0 = const()[name = tensor("matmul_9_transpose_y_0"), val = tensor(true)]; + tensor matmul_9_transpose_x_0 = const()[name = tensor("matmul_9_transpose_x_0"), val = tensor(false)]; + tensor transpose_90_perm_0 = const()[name = tensor("transpose_90_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_91_perm_0 = const()[name = tensor("transpose_91_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_91 = transpose(perm = transpose_91_perm_0, x = var_823_cast_fp16)[name = tensor("transpose_131")]; + tensor transpose_90 = transpose(perm = transpose_90_perm_0, x = mul_9_cast_fp16)[name = tensor("transpose_132")]; + tensor matmul_9_cast_fp16 = matmul(transpose_x = matmul_9_transpose_x_0, transpose_y = matmul_9_transpose_y_0, x = transpose_90, y = transpose_91)[name = tensor("matmul_9_cast_fp16")]; + tensor softmax_9_axis_0 = const()[name = tensor("softmax_9_axis_0"), val = tensor(-1)]; + tensor softmax_9_cast_fp16 = softmax(axis = softmax_9_axis_0, x = matmul_9_cast_fp16)[name = tensor("softmax_9_cast_fp16")]; + tensor attention_output_37_transpose_x_0 = const()[name = tensor("attention_output_37_transpose_x_0"), val = tensor(false)]; + tensor attention_output_37_transpose_y_0 = const()[name = tensor("attention_output_37_transpose_y_0"), val = tensor(false)]; + tensor value_layer_19_cast_fp16 = transpose(perm = value_layer_19_perm_0, x = var_829_cast_fp16)[name = tensor("transpose_133")]; + tensor attention_output_37_cast_fp16 = matmul(transpose_x = attention_output_37_transpose_x_0, transpose_y = attention_output_37_transpose_y_0, x = softmax_9_cast_fp16, y = value_layer_19_cast_fp16)[name = tensor("attention_output_37_cast_fp16")]; + tensor attention_output_39_perm_0 = const()[name = tensor("attention_output_39_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_833 = const()[name = tensor("op_833"), val = tensor([1, 256, 768])]; + tensor attention_output_39_cast_fp16 = transpose(perm = attention_output_39_perm_0, x = attention_output_37_cast_fp16)[name = tensor("transpose_130")]; + tensor input_163_cast_fp16 = reshape(shape = var_833, x = attention_output_39_cast_fp16)[name = tensor("input_163_cast_fp16")]; + tensor linear_58_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_163_cast_fp16)[name = tensor("linear_58_cast_fp16")]; + tensor input_167_cast_fp16 = add(x = hidden_states_19_cast_fp16, y = linear_58_cast_fp16)[name = tensor("input_167_cast_fp16")]; + tensor input_169_axes_0 = const()[name = tensor("input_169_axes_0"), val = tensor([-1])]; + tensor input_169_cast_fp16 = layer_norm(axes = input_169_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_167_cast_fp16)[name = tensor("input_169_cast_fp16")]; + tensor linear_59_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_169_cast_fp16)[name = tensor("linear_59_cast_fp16")]; + tensor input_173_mode_0 = const()[name = tensor("input_173_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_173_cast_fp16 = gelu(mode = input_173_mode_0, x = linear_59_cast_fp16)[name = tensor("input_173_cast_fp16")]; + tensor linear_60_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_173_cast_fp16)[name = tensor("linear_60_cast_fp16")]; + tensor input_175_cast_fp16 = add(x = linear_60_cast_fp16, y = input_169_cast_fp16)[name = tensor("input_175_cast_fp16")]; + tensor hidden_states_21_axes_0 = const()[name = tensor("hidden_states_21_axes_0"), val = tensor([-1])]; + tensor hidden_states_21_cast_fp16 = layer_norm(axes = hidden_states_21_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_175_cast_fp16)[name = tensor("hidden_states_21_cast_fp16")]; + tensor linear_61_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_21_cast_fp16)[name = tensor("linear_61_cast_fp16")]; + tensor var_883 = const()[name = tensor("op_883"), val = tensor([1, -1, 12, 64])]; + tensor var_884_cast_fp16 = reshape(shape = var_883, x = linear_61_cast_fp16)[name = tensor("op_884_cast_fp16")]; + tensor linear_62_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_21_cast_fp16)[name = tensor("linear_62_cast_fp16")]; + tensor var_889 = const()[name = tensor("op_889"), val = tensor([1, -1, 12, 64])]; + tensor var_890_cast_fp16 = reshape(shape = var_889, x = linear_62_cast_fp16)[name = tensor("op_890_cast_fp16")]; + tensor linear_63_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_21_cast_fp16)[name = tensor("linear_63_cast_fp16")]; + tensor var_895 = const()[name = tensor("op_895"), val = tensor([1, -1, 12, 64])]; + tensor var_896_cast_fp16 = reshape(shape = var_895, x = linear_63_cast_fp16)[name = tensor("op_896_cast_fp16")]; + tensor value_layer_21_perm_0 = const()[name = tensor("value_layer_21_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_10_y_0_to_fp16 = const()[name = tensor("mul_10_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_10_cast_fp16 = mul(x = var_884_cast_fp16, y = mul_10_y_0_to_fp16)[name = tensor("mul_10_cast_fp16")]; + tensor matmul_10_transpose_y_0 = const()[name = tensor("matmul_10_transpose_y_0"), val = tensor(true)]; + tensor matmul_10_transpose_x_0 = const()[name = tensor("matmul_10_transpose_x_0"), val = tensor(false)]; + tensor transpose_92_perm_0 = const()[name = tensor("transpose_92_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_93_perm_0 = const()[name = tensor("transpose_93_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_93 = transpose(perm = transpose_93_perm_0, x = var_890_cast_fp16)[name = tensor("transpose_127")]; + tensor transpose_92 = transpose(perm = transpose_92_perm_0, x = mul_10_cast_fp16)[name = tensor("transpose_128")]; + tensor matmul_10_cast_fp16 = matmul(transpose_x = matmul_10_transpose_x_0, transpose_y = matmul_10_transpose_y_0, x = transpose_92, y = transpose_93)[name = tensor("matmul_10_cast_fp16")]; + tensor softmax_10_axis_0 = const()[name = tensor("softmax_10_axis_0"), val = tensor(-1)]; + tensor softmax_10_cast_fp16 = softmax(axis = softmax_10_axis_0, x = matmul_10_cast_fp16)[name = tensor("softmax_10_cast_fp16")]; + tensor attention_output_41_transpose_x_0 = const()[name = tensor("attention_output_41_transpose_x_0"), val = tensor(false)]; + tensor attention_output_41_transpose_y_0 = const()[name = tensor("attention_output_41_transpose_y_0"), val = tensor(false)]; + tensor value_layer_21_cast_fp16 = transpose(perm = value_layer_21_perm_0, x = var_896_cast_fp16)[name = tensor("transpose_129")]; + tensor attention_output_41_cast_fp16 = matmul(transpose_x = attention_output_41_transpose_x_0, transpose_y = attention_output_41_transpose_y_0, x = softmax_10_cast_fp16, y = value_layer_21_cast_fp16)[name = tensor("attention_output_41_cast_fp16")]; + tensor attention_output_43_perm_0 = const()[name = tensor("attention_output_43_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_900 = const()[name = tensor("op_900"), val = tensor([1, 256, 768])]; + tensor attention_output_43_cast_fp16 = transpose(perm = attention_output_43_perm_0, x = attention_output_41_cast_fp16)[name = tensor("transpose_126")]; + tensor input_177_cast_fp16 = reshape(shape = var_900, x = attention_output_43_cast_fp16)[name = tensor("input_177_cast_fp16")]; + tensor linear_64_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_177_cast_fp16)[name = tensor("linear_64_cast_fp16")]; + tensor input_181_cast_fp16 = add(x = hidden_states_21_cast_fp16, y = linear_64_cast_fp16)[name = tensor("input_181_cast_fp16")]; + tensor input_183_axes_0 = const()[name = tensor("input_183_axes_0"), val = tensor([-1])]; + tensor input_183_cast_fp16 = layer_norm(axes = input_183_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_181_cast_fp16)[name = tensor("input_183_cast_fp16")]; + tensor linear_65_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_183_cast_fp16)[name = tensor("linear_65_cast_fp16")]; + tensor input_187_mode_0 = const()[name = tensor("input_187_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_187_cast_fp16 = gelu(mode = input_187_mode_0, x = linear_65_cast_fp16)[name = tensor("input_187_cast_fp16")]; + tensor linear_66_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_187_cast_fp16)[name = tensor("linear_66_cast_fp16")]; + tensor input_189_cast_fp16 = add(x = linear_66_cast_fp16, y = input_183_cast_fp16)[name = tensor("input_189_cast_fp16")]; + tensor hidden_states_axes_0 = const()[name = tensor("hidden_states_axes_0"), val = tensor([-1])]; + tensor hidden_states_cast_fp16 = layer_norm(axes = hidden_states_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_189_cast_fp16)[name = tensor("hidden_states_cast_fp16")]; + tensor linear_67_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_cast_fp16)[name = tensor("linear_67_cast_fp16")]; + tensor var_950 = const()[name = tensor("op_950"), val = tensor([1, -1, 12, 64])]; + tensor var_951_cast_fp16 = reshape(shape = var_950, x = linear_67_cast_fp16)[name = tensor("op_951_cast_fp16")]; + tensor linear_68_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_cast_fp16)[name = tensor("linear_68_cast_fp16")]; + tensor var_956 = const()[name = tensor("op_956"), val = tensor([1, -1, 12, 64])]; + tensor var_957_cast_fp16 = reshape(shape = var_956, x = linear_68_cast_fp16)[name = tensor("op_957_cast_fp16")]; + tensor linear_69_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_cast_fp16)[name = tensor("linear_69_cast_fp16")]; + tensor var_962 = const()[name = tensor("op_962"), val = tensor([1, -1, 12, 64])]; + tensor var_963_cast_fp16 = reshape(shape = var_962, x = linear_69_cast_fp16)[name = tensor("op_963_cast_fp16")]; + tensor value_layer_perm_0 = const()[name = tensor("value_layer_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor mul_11_y_0_to_fp16 = const()[name = tensor("mul_11_y_0_to_fp16"), val = tensor(0x1p-3)]; + tensor mul_11_cast_fp16 = mul(x = var_951_cast_fp16, y = mul_11_y_0_to_fp16)[name = tensor("mul_11_cast_fp16")]; + tensor matmul_11_transpose_y_0 = const()[name = tensor("matmul_11_transpose_y_0"), val = tensor(true)]; + tensor matmul_11_transpose_x_0 = const()[name = tensor("matmul_11_transpose_x_0"), val = tensor(false)]; + tensor transpose_94_perm_0 = const()[name = tensor("transpose_94_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_95_perm_0 = const()[name = tensor("transpose_95_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_95 = transpose(perm = transpose_95_perm_0, x = var_957_cast_fp16)[name = tensor("transpose_123")]; + tensor transpose_94 = transpose(perm = transpose_94_perm_0, x = mul_11_cast_fp16)[name = tensor("transpose_124")]; + tensor matmul_11_cast_fp16 = matmul(transpose_x = matmul_11_transpose_x_0, transpose_y = matmul_11_transpose_y_0, x = transpose_94, y = transpose_95)[name = tensor("matmul_11_cast_fp16")]; + tensor softmax_11_axis_0 = const()[name = tensor("softmax_11_axis_0"), val = tensor(-1)]; + tensor softmax_11_cast_fp16 = softmax(axis = softmax_11_axis_0, x = matmul_11_cast_fp16)[name = tensor("softmax_11_cast_fp16")]; + tensor attention_output_45_transpose_x_0 = const()[name = tensor("attention_output_45_transpose_x_0"), val = tensor(false)]; + tensor attention_output_45_transpose_y_0 = const()[name = tensor("attention_output_45_transpose_y_0"), val = tensor(false)]; + tensor value_layer_cast_fp16 = transpose(perm = value_layer_perm_0, x = var_963_cast_fp16)[name = tensor("transpose_125")]; + tensor attention_output_45_cast_fp16 = matmul(transpose_x = attention_output_45_transpose_x_0, transpose_y = attention_output_45_transpose_y_0, x = softmax_11_cast_fp16, y = value_layer_cast_fp16)[name = tensor("attention_output_45_cast_fp16")]; + tensor attention_output_perm_0 = const()[name = tensor("attention_output_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_967 = const()[name = tensor("op_967"), val = tensor([1, 256, 768])]; + tensor attention_output_cast_fp16 = transpose(perm = attention_output_perm_0, x = attention_output_45_cast_fp16)[name = tensor("transpose_122")]; + tensor input_191_cast_fp16 = reshape(shape = var_967, x = attention_output_cast_fp16)[name = tensor("input_191_cast_fp16")]; + tensor linear_70_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_191_cast_fp16)[name = tensor("linear_70_cast_fp16")]; + tensor input_195_cast_fp16 = add(x = hidden_states_cast_fp16, y = linear_70_cast_fp16)[name = tensor("input_195_cast_fp16")]; + tensor input_197_axes_0 = const()[name = tensor("input_197_axes_0"), val = tensor([-1])]; + tensor input_197_cast_fp16 = layer_norm(axes = input_197_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_195_cast_fp16)[name = tensor("input_197_cast_fp16")]; + tensor linear_71_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_197_cast_fp16)[name = tensor("linear_71_cast_fp16")]; + tensor input_201_mode_0 = const()[name = tensor("input_201_mode_0"), val = tensor("TANH_APPROXIMATION")]; + tensor input_201_cast_fp16 = gelu(mode = input_201_mode_0, x = linear_71_cast_fp16)[name = tensor("input_201_cast_fp16")]; + tensor linear_72_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_201_cast_fp16)[name = tensor("linear_72_cast_fp16")]; + tensor input_203_cast_fp16 = add(x = linear_72_cast_fp16, y = input_197_cast_fp16)[name = tensor("input_203_cast_fp16")]; + tensor sequence_output_axes_0 = const()[name = tensor("sequence_output_axes_0"), val = tensor([-1])]; + tensor bert_dur = layer_norm(axes = sequence_output_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_131_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_203_cast_fp16)[name = tensor("sequence_output_cast_fp16")]; + tensor bert_encoder_weight_to_fp16 = const()[name = tensor("bert_encoder_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(22612736)))]; + tensor bert_encoder_bias_to_fp16 = const()[name = tensor("bert_encoder_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(23399232)))]; + tensor linear_73_cast_fp16 = linear(bias = bert_encoder_bias_to_fp16, weight = bert_encoder_weight_to_fp16, x = bert_dur)[name = tensor("linear_73_cast_fp16")]; + tensor x_19_perm_0 = const()[name = tensor("x_19_perm_0"), val = tensor([0, -1, -2])]; + tensor var_1017 = const()[name = tensor("op_1017"), val = tensor(-1)]; + tensor var_1018 = const()[name = tensor("op_1018"), val = tensor(1)]; + tensor var_1033 = const()[name = tensor("op_1033"), val = tensor([2, 0, 1])]; + tensor expand_dims_0_axes_0 = const()[name = tensor("expand_dims_0_axes_0"), val = tensor([0])]; + tensor style_to_fp16_dtype_0 = const()[name = tensor("style_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor style_to_fp16 = cast(dtype = style_to_fp16_dtype_0, x = style)[name = tensor("cast_62")]; + tensor expand_dims_0_cast_fp16 = expand_dims(axes = expand_dims_0_axes_0, x = style_to_fp16)[name = tensor("expand_dims_0_cast_fp16")]; + tensor s_reps_0 = const()[name = tensor("s_reps_0"), val = tensor([256, 1, 1])]; + tensor s_cast_fp16 = tile(reps = s_reps_0, x = expand_dims_0_cast_fp16)[name = tensor("s_cast_fp16")]; + tensor x_23_interleave_0 = const()[name = tensor("x_23_interleave_0"), val = tensor(false)]; + tensor d_en = transpose(perm = x_19_perm_0, x = linear_73_cast_fp16)[name = tensor("transpose_121")]; + tensor x_21_cast_fp16 = transpose(perm = var_1033, x = d_en)[name = tensor("transpose_120")]; + tensor x_23_cast_fp16 = concat(axis = var_1017, interleave = x_23_interleave_0, values = (x_21_cast_fp16, s_cast_fp16))[name = tensor("x_23_cast_fp16")]; + tensor x_29_batch_first_direction_0 = const()[name = tensor("x_29_batch_first_direction_0"), val = tensor("bidirectional")]; + tensor x_29_batch_first_output_sequence_0 = const()[name = tensor("x_29_batch_first_output_sequence_0"), val = tensor(true)]; + tensor x_29_batch_first_recurrent_activation_0 = const()[name = tensor("x_29_batch_first_recurrent_activation_0"), val = tensor("sigmoid")]; + tensor x_29_batch_first_cell_activation_0 = const()[name = tensor("x_29_batch_first_cell_activation_0"), val = tensor("tanh")]; + tensor x_29_batch_first_activation_0 = const()[name = tensor("x_29_batch_first_activation_0"), val = tensor("tanh")]; + tensor concat_17_to_fp16 = const()[name = tensor("concat_17_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(23400320)))]; + tensor concat_18_to_fp16 = const()[name = tensor("concat_18_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(24711104)))]; + tensor add_14_to_fp16 = const()[name = tensor("add_14_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(25235456)))]; + tensor concat_19_to_fp16 = const()[name = tensor("concat_19_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(25237568)))]; + tensor concat_20_to_fp16 = const()[name = tensor("concat_20_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(26548352)))]; + tensor add_15_to_fp16 = const()[name = tensor("add_15_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(27072704)))]; + tensor x_29_batch_first_cast_fp16_0, tensor x_29_batch_first_cast_fp16_1, tensor x_29_batch_first_cast_fp16_2 = lstm(activation = x_29_batch_first_activation_0, bias = add_14_to_fp16, bias_back = add_15_to_fp16, cell_activation = x_29_batch_first_cell_activation_0, direction = x_29_batch_first_direction_0, initial_c = x_17_batch_first_lstm_h0_reshaped_to_fp16, initial_h = x_17_batch_first_lstm_h0_reshaped_to_fp16, output_sequence = x_29_batch_first_output_sequence_0, recurrent_activation = x_29_batch_first_recurrent_activation_0, weight_hh = concat_18_to_fp16, weight_hh_back = concat_20_to_fp16, weight_ih = concat_17_to_fp16, weight_ih_back = concat_19_to_fp16, x = x_23_cast_fp16)[name = tensor("x_29_batch_first_cast_fp16")]; + tensor transpose_39_perm_0 = const()[name = tensor("transpose_39_perm_0"), val = tensor([1, 0, 2])]; + tensor duration_encoder_lstms_1_fc_weight_to_fp16 = const()[name = tensor("duration_encoder_lstms_1_fc_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(27074816)))]; + tensor duration_encoder_lstms_1_fc_bias_to_fp16 = const()[name = tensor("duration_encoder_lstms_1_fc_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(27337024)))]; + tensor linear_74_cast_fp16 = linear(bias = duration_encoder_lstms_1_fc_bias_to_fp16, weight = duration_encoder_lstms_1_fc_weight_to_fp16, x = style_to_fp16)[name = tensor("linear_74_cast_fp16")]; + tensor var_1072 = const()[name = tensor("op_1072"), val = tensor([1, 1024, 1])]; + tensor h_3_cast_fp16 = reshape(shape = var_1072, x = linear_74_cast_fp16)[name = tensor("h_3_cast_fp16")]; + tensor var_1074_split_sizes_0 = const()[name = tensor("op_1074_split_sizes_0"), val = tensor([512, 512])]; + tensor var_1074_axis_0 = const()[name = tensor("op_1074_axis_0"), val = tensor(1)]; + tensor var_1074_cast_fp16_0, tensor var_1074_cast_fp16_1 = split(axis = var_1074_axis_0, split_sizes = var_1074_split_sizes_0, x = h_3_cast_fp16)[name = tensor("op_1074_cast_fp16")]; + tensor gamma_3_perm_0 = const()[name = tensor("gamma_3_perm_0"), val = tensor([0, -1, 1])]; + tensor beta_3_perm_0 = const()[name = tensor("beta_3_perm_0"), val = tensor([0, -1, 1])]; + tensor x_37_axes_0 = const()[name = tensor("x_37_axes_0"), val = tensor([-1])]; + tensor var_1008_to_fp16 = const()[name = tensor("op_1008_to_fp16"), val = tensor(0x1.5p-17)]; + tensor transpose_39_cast_fp16 = transpose(perm = transpose_39_perm_0, x = x_29_batch_first_cast_fp16_0)[name = tensor("transpose_119")]; + tensor x_37_cast_fp16 = layer_norm(axes = x_37_axes_0, epsilon = var_1008_to_fp16, x = transpose_39_cast_fp16)[name = tensor("x_37_cast_fp16")]; + tensor var_1080_promoted_to_fp16 = const()[name = tensor("op_1080_promoted_to_fp16"), val = tensor(0x1p+0)]; + tensor gamma_3_cast_fp16 = transpose(perm = gamma_3_perm_0, x = var_1074_cast_fp16_0)[name = tensor("transpose_118")]; + tensor var_1081_cast_fp16 = add(x = gamma_3_cast_fp16, y = var_1080_promoted_to_fp16)[name = tensor("op_1081_cast_fp16")]; + tensor var_1082_cast_fp16 = mul(x = var_1081_cast_fp16, y = x_37_cast_fp16)[name = tensor("op_1082_cast_fp16")]; + tensor beta_3_cast_fp16 = transpose(perm = beta_3_perm_0, x = var_1074_cast_fp16_1)[name = tensor("transpose_117")]; + tensor x_39_cast_fp16 = add(x = var_1082_cast_fp16, y = beta_3_cast_fp16)[name = tensor("x_39_cast_fp16")]; + tensor x_43_interleave_0 = const()[name = tensor("x_43_interleave_0"), val = tensor(false)]; + tensor transpose_96_perm_0 = const()[name = tensor("transpose_96_perm_0"), val = tensor([0, -1, -2])]; + tensor transpose_97_perm_0 = const()[name = tensor("transpose_97_perm_0"), val = tensor([1, 2, 0])]; + tensor transpose_97 = transpose(perm = transpose_97_perm_0, x = s_cast_fp16)[name = tensor("transpose_115")]; + tensor transpose_96 = transpose(perm = transpose_96_perm_0, x = x_39_cast_fp16)[name = tensor("transpose_116")]; + tensor x_43_cast_fp16 = concat(axis = var_1018, interleave = x_43_interleave_0, values = (transpose_96, transpose_97))[name = tensor("x_43_cast_fp16")]; + tensor transpose_34_perm_0 = const()[name = tensor("transpose_34_perm_0"), val = tensor([-1, 0, -2])]; + tensor x_45_batch_first_direction_0 = const()[name = tensor("x_45_batch_first_direction_0"), val = tensor("bidirectional")]; + tensor x_45_batch_first_output_sequence_0 = const()[name = tensor("x_45_batch_first_output_sequence_0"), val = tensor(true)]; + tensor x_45_batch_first_recurrent_activation_0 = const()[name = tensor("x_45_batch_first_recurrent_activation_0"), val = tensor("sigmoid")]; + tensor x_45_batch_first_cell_activation_0 = const()[name = tensor("x_45_batch_first_cell_activation_0"), val = tensor("tanh")]; + tensor x_45_batch_first_activation_0 = const()[name = tensor("x_45_batch_first_activation_0"), val = tensor("tanh")]; + tensor concat_27_to_fp16 = const()[name = tensor("concat_27_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(27339136)))]; + tensor concat_28_to_fp16 = const()[name = tensor("concat_28_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(28649920)))]; + tensor add_16_to_fp16 = const()[name = tensor("add_16_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(29174272)))]; + tensor concat_29_to_fp16 = const()[name = tensor("concat_29_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(29176384)))]; + tensor concat_30_to_fp16 = const()[name = tensor("concat_30_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(30487168)))]; + tensor add_17_to_fp16 = const()[name = tensor("add_17_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(31011520)))]; + tensor transpose_34_cast_fp16 = transpose(perm = transpose_34_perm_0, x = x_43_cast_fp16)[name = tensor("transpose_114")]; + tensor x_45_batch_first_cast_fp16_0, tensor x_45_batch_first_cast_fp16_1, tensor x_45_batch_first_cast_fp16_2 = lstm(activation = x_45_batch_first_activation_0, bias = add_16_to_fp16, bias_back = add_17_to_fp16, cell_activation = x_45_batch_first_cell_activation_0, direction = x_45_batch_first_direction_0, initial_c = x_17_batch_first_lstm_h0_reshaped_to_fp16, initial_h = x_17_batch_first_lstm_h0_reshaped_to_fp16, output_sequence = x_45_batch_first_output_sequence_0, recurrent_activation = x_45_batch_first_recurrent_activation_0, weight_hh = concat_28_to_fp16, weight_hh_back = concat_30_to_fp16, weight_ih = concat_27_to_fp16, weight_ih_back = concat_29_to_fp16, x = transpose_34_cast_fp16)[name = tensor("x_45_batch_first_cast_fp16")]; + tensor transpose_40_perm_0 = const()[name = tensor("transpose_40_perm_0"), val = tensor([1, 0, 2])]; + tensor duration_encoder_lstms_3_fc_weight_to_fp16 = const()[name = tensor("duration_encoder_lstms_3_fc_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(31013632)))]; + tensor duration_encoder_lstms_3_fc_bias_to_fp16 = const()[name = tensor("duration_encoder_lstms_3_fc_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(31275840)))]; + tensor linear_75_cast_fp16 = linear(bias = duration_encoder_lstms_3_fc_bias_to_fp16, weight = duration_encoder_lstms_3_fc_weight_to_fp16, x = style_to_fp16)[name = tensor("linear_75_cast_fp16")]; + tensor var_1120 = const()[name = tensor("op_1120"), val = tensor([1, 1024, 1])]; + tensor h_7_cast_fp16 = reshape(shape = var_1120, x = linear_75_cast_fp16)[name = tensor("h_7_cast_fp16")]; + tensor var_1122_split_sizes_0 = const()[name = tensor("op_1122_split_sizes_0"), val = tensor([512, 512])]; + tensor var_1122_axis_0 = const()[name = tensor("op_1122_axis_0"), val = tensor(1)]; + tensor var_1122_cast_fp16_0, tensor var_1122_cast_fp16_1 = split(axis = var_1122_axis_0, split_sizes = var_1122_split_sizes_0, x = h_7_cast_fp16)[name = tensor("op_1122_cast_fp16")]; + tensor gamma_7_perm_0 = const()[name = tensor("gamma_7_perm_0"), val = tensor([0, -1, 1])]; + tensor beta_7_perm_0 = const()[name = tensor("beta_7_perm_0"), val = tensor([0, -1, 1])]; + tensor x_53_axes_0 = const()[name = tensor("x_53_axes_0"), val = tensor([-1])]; + tensor transpose_40_cast_fp16 = transpose(perm = transpose_40_perm_0, x = x_45_batch_first_cast_fp16_0)[name = tensor("transpose_113")]; + tensor x_53_cast_fp16 = layer_norm(axes = x_53_axes_0, epsilon = var_1008_to_fp16, x = transpose_40_cast_fp16)[name = tensor("x_53_cast_fp16")]; + tensor var_1128_promoted_to_fp16 = const()[name = tensor("op_1128_promoted_to_fp16"), val = tensor(0x1p+0)]; + tensor gamma_7_cast_fp16 = transpose(perm = gamma_7_perm_0, x = var_1122_cast_fp16_0)[name = tensor("transpose_112")]; + tensor var_1129_cast_fp16 = add(x = gamma_7_cast_fp16, y = var_1128_promoted_to_fp16)[name = tensor("op_1129_cast_fp16")]; + tensor var_1130_cast_fp16 = mul(x = var_1129_cast_fp16, y = x_53_cast_fp16)[name = tensor("op_1130_cast_fp16")]; + tensor beta_7_cast_fp16 = transpose(perm = beta_7_perm_0, x = var_1122_cast_fp16_1)[name = tensor("transpose_111")]; + tensor x_55_cast_fp16 = add(x = var_1130_cast_fp16, y = beta_7_cast_fp16)[name = tensor("x_55_cast_fp16")]; + tensor x_59_interleave_0 = const()[name = tensor("x_59_interleave_0"), val = tensor(false)]; + tensor transpose_100_perm_0 = const()[name = tensor("transpose_100_perm_0"), val = tensor([0, -1, -2])]; + tensor transpose_100 = transpose(perm = transpose_100_perm_0, x = x_55_cast_fp16)[name = tensor("transpose_110")]; + tensor x_59_cast_fp16 = concat(axis = var_1018, interleave = x_59_interleave_0, values = (transpose_100, transpose_97))[name = tensor("x_59_cast_fp16")]; + tensor transpose_36_perm_0 = const()[name = tensor("transpose_36_perm_0"), val = tensor([-1, 0, -2])]; + tensor x_61_batch_first_direction_0 = const()[name = tensor("x_61_batch_first_direction_0"), val = tensor("bidirectional")]; + tensor x_61_batch_first_output_sequence_0 = const()[name = tensor("x_61_batch_first_output_sequence_0"), val = tensor(true)]; + tensor x_61_batch_first_recurrent_activation_0 = const()[name = tensor("x_61_batch_first_recurrent_activation_0"), val = tensor("sigmoid")]; + tensor x_61_batch_first_cell_activation_0 = const()[name = tensor("x_61_batch_first_cell_activation_0"), val = tensor("tanh")]; + tensor x_61_batch_first_activation_0 = const()[name = tensor("x_61_batch_first_activation_0"), val = tensor("tanh")]; + tensor concat_37_to_fp16 = const()[name = tensor("concat_37_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(31277952)))]; + tensor concat_38_to_fp16 = const()[name = tensor("concat_38_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(32588736)))]; + tensor add_18_to_fp16 = const()[name = tensor("add_18_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(33113088)))]; + tensor concat_39_to_fp16 = const()[name = tensor("concat_39_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(33115200)))]; + tensor concat_40_to_fp16 = const()[name = tensor("concat_40_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(34425984)))]; + tensor add_19_to_fp16 = const()[name = tensor("add_19_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(34950336)))]; + tensor transpose_36_cast_fp16 = transpose(perm = transpose_36_perm_0, x = x_59_cast_fp16)[name = tensor("transpose_109")]; + tensor x_61_batch_first_cast_fp16_0, tensor x_61_batch_first_cast_fp16_1, tensor x_61_batch_first_cast_fp16_2 = lstm(activation = x_61_batch_first_activation_0, bias = add_18_to_fp16, bias_back = add_19_to_fp16, cell_activation = x_61_batch_first_cell_activation_0, direction = x_61_batch_first_direction_0, initial_c = x_17_batch_first_lstm_h0_reshaped_to_fp16, initial_h = x_17_batch_first_lstm_h0_reshaped_to_fp16, output_sequence = x_61_batch_first_output_sequence_0, recurrent_activation = x_61_batch_first_recurrent_activation_0, weight_hh = concat_38_to_fp16, weight_hh_back = concat_40_to_fp16, weight_ih = concat_37_to_fp16, weight_ih_back = concat_39_to_fp16, x = transpose_36_cast_fp16)[name = tensor("x_61_batch_first_cast_fp16")]; + tensor transpose_41_perm_0 = const()[name = tensor("transpose_41_perm_0"), val = tensor([1, 0, 2])]; + tensor duration_encoder_lstms_5_fc_weight_to_fp16 = const()[name = tensor("duration_encoder_lstms_5_fc_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(34952448)))]; + tensor duration_encoder_lstms_5_fc_bias_to_fp16 = const()[name = tensor("duration_encoder_lstms_5_fc_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(35214656)))]; + tensor linear_76_cast_fp16 = linear(bias = duration_encoder_lstms_5_fc_bias_to_fp16, weight = duration_encoder_lstms_5_fc_weight_to_fp16, x = style_to_fp16)[name = tensor("linear_76_cast_fp16")]; + tensor var_1168 = const()[name = tensor("op_1168"), val = tensor([1, 1024, 1])]; + tensor h_cast_fp16 = reshape(shape = var_1168, x = linear_76_cast_fp16)[name = tensor("h_cast_fp16")]; + tensor var_1170_split_sizes_0 = const()[name = tensor("op_1170_split_sizes_0"), val = tensor([512, 512])]; + tensor var_1170_axis_0 = const()[name = tensor("op_1170_axis_0"), val = tensor(1)]; + tensor var_1170_cast_fp16_0, tensor var_1170_cast_fp16_1 = split(axis = var_1170_axis_0, split_sizes = var_1170_split_sizes_0, x = h_cast_fp16)[name = tensor("op_1170_cast_fp16")]; + tensor gamma_11_perm_0 = const()[name = tensor("gamma_11_perm_0"), val = tensor([0, -1, 1])]; + tensor beta_11_perm_0 = const()[name = tensor("beta_11_perm_0"), val = tensor([0, -1, 1])]; + tensor x_69_axes_0 = const()[name = tensor("x_69_axes_0"), val = tensor([-1])]; + tensor transpose_41_cast_fp16 = transpose(perm = transpose_41_perm_0, x = x_61_batch_first_cast_fp16_0)[name = tensor("transpose_108")]; + tensor x_69_cast_fp16 = layer_norm(axes = x_69_axes_0, epsilon = var_1008_to_fp16, x = transpose_41_cast_fp16)[name = tensor("x_69_cast_fp16")]; + tensor var_1176_promoted_to_fp16 = const()[name = tensor("op_1176_promoted_to_fp16"), val = tensor(0x1p+0)]; + tensor gamma_11_cast_fp16 = transpose(perm = gamma_11_perm_0, x = var_1170_cast_fp16_0)[name = tensor("transpose_107")]; + tensor var_1177_cast_fp16 = add(x = gamma_11_cast_fp16, y = var_1176_promoted_to_fp16)[name = tensor("op_1177_cast_fp16")]; + tensor var_1178_cast_fp16 = mul(x = var_1177_cast_fp16, y = x_69_cast_fp16)[name = tensor("op_1178_cast_fp16")]; + tensor beta_11_cast_fp16 = transpose(perm = beta_11_perm_0, x = var_1170_cast_fp16_1)[name = tensor("transpose_106")]; + tensor x_71_cast_fp16 = add(x = var_1178_cast_fp16, y = beta_11_cast_fp16)[name = tensor("x_71_cast_fp16")]; + tensor x_75_interleave_0 = const()[name = tensor("x_75_interleave_0"), val = tensor(false)]; + tensor transpose_101_perm_0 = const()[name = tensor("transpose_101_perm_0"), val = tensor([0, -1, -2])]; + tensor transpose_101 = transpose(perm = transpose_101_perm_0, x = x_71_cast_fp16)[name = tensor("transpose_105")]; + tensor x_75_cast_fp16 = concat(axis = var_1018, interleave = x_75_interleave_0, values = (transpose_101, transpose_97))[name = tensor("x_75_cast_fp16")]; + tensor input_221_perm_0 = const()[name = tensor("input_221_perm_0"), val = tensor([0, -1, -2])]; + tensor input_221_batch_first_transpose_perm_0 = const()[name = tensor("input_221_batch_first_transpose_perm_0"), val = tensor([1, 0, 2])]; + tensor input_223_batch_first_direction_0 = const()[name = tensor("input_223_batch_first_direction_0"), val = tensor("bidirectional")]; + tensor input_223_batch_first_output_sequence_0 = const()[name = tensor("input_223_batch_first_output_sequence_0"), val = tensor(true)]; + tensor input_223_batch_first_recurrent_activation_0 = const()[name = tensor("input_223_batch_first_recurrent_activation_0"), val = tensor("sigmoid")]; + tensor input_223_batch_first_cell_activation_0 = const()[name = tensor("input_223_batch_first_cell_activation_0"), val = tensor("tanh")]; + tensor input_223_batch_first_activation_0 = const()[name = tensor("input_223_batch_first_activation_0"), val = tensor("tanh")]; + tensor concat_47_to_fp16 = const()[name = tensor("concat_47_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(35216768)))]; + tensor concat_48_to_fp16 = const()[name = tensor("concat_48_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(36527552)))]; + tensor add_20_to_fp16 = const()[name = tensor("add_20_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(37051904)))]; + tensor concat_49_to_fp16 = const()[name = tensor("concat_49_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(37054016)))]; + tensor concat_50_to_fp16 = const()[name = tensor("concat_50_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(38364800)))]; + tensor add_21_to_fp16 = const()[name = tensor("add_21_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(38889152)))]; + tensor d = transpose(perm = input_221_perm_0, x = x_75_cast_fp16)[name = tensor("transpose_104")]; + tensor input_221_batch_first_transpose_cast_fp16 = transpose(perm = input_221_batch_first_transpose_perm_0, x = d)[name = tensor("transpose_103")]; + tensor input_223_batch_first_cast_fp16_0, tensor input_223_batch_first_cast_fp16_1, tensor input_223_batch_first_cast_fp16_2 = lstm(activation = input_223_batch_first_activation_0, bias = add_20_to_fp16, bias_back = add_21_to_fp16, cell_activation = input_223_batch_first_cell_activation_0, direction = input_223_batch_first_direction_0, initial_c = x_17_batch_first_lstm_h0_reshaped_to_fp16, initial_h = x_17_batch_first_lstm_h0_reshaped_to_fp16, output_sequence = input_223_batch_first_output_sequence_0, recurrent_activation = input_223_batch_first_recurrent_activation_0, weight_hh = concat_48_to_fp16, weight_hh_back = concat_50_to_fp16, weight_ih = concat_47_to_fp16, weight_ih_back = concat_49_to_fp16, x = input_221_batch_first_transpose_cast_fp16)[name = tensor("input_223_batch_first_cast_fp16")]; + tensor input_223_perm_0 = const()[name = tensor("input_223_perm_0"), val = tensor([1, 0, 2])]; + tensor duration_proj_linear_layer_weight_to_fp16 = const()[name = tensor("duration_proj_linear_layer_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(38891264)))]; + tensor duration_proj_linear_layer_bias_to_fp16 = const()[name = tensor("duration_proj_linear_layer_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(38942528)))]; + tensor input_223_cast_fp16 = transpose(perm = input_223_perm_0, x = input_223_batch_first_cast_fp16_0)[name = tensor("transpose_102")]; + tensor pred_dur_log = linear(bias = duration_proj_linear_layer_bias_to_fp16, weight = duration_proj_linear_layer_weight_to_fp16, x = input_223_cast_fp16)[name = tensor("linear_77_cast_fp16")]; + tensor fixed_embedding = const()[name = tensor("fixed_embedding"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(38942720)))]; + tensor t_en = transpose(perm = transpose_30_perm_0, x = x_17_batch_first_cast_fp16_0)[name = tensor("transpose_170")]; + } -> (t_en, d_en, d, pred_dur_log, fixed_embedding, bert_dur); +} \ No newline at end of file