03 Ranking

# 遍历所有需要交叉的特征对
<div markdown="1" style="margin-top: -30px; font-size: 0.75em; opacity: 0.7;">
:material-circle-edit-outline: 约 9189 个字 :fontawesome-solid-code: 374 行代码 :material-image-multiple-outline: 13 张图片 :material-clock-time-two-outline: 预计阅读时间 35 分钟
</div>
for i in range(len(cross_feature_columns)):
    for j in range(i + 1, len(cross_feature_columns)):
        fc_i = cross_feature_columns[i]
        fc_j = cross_feature_columns[j]

        # 获取两个特征的输入
        feat_i = input_layer_dict[fc_i.name]  # [B, 1]
        feat_j = input_layer_dict[fc_j.name]  # [B, 1]

        # 为每个特征对创建独立的权重表
        cross_vocab_size = fc_i.vocab_size * fc_j.vocab_size
        cross_embedding = Embedding(
            input_dim=cross_vocab_size,
            output_dim=1,  # 标量权重，直接记住这对特征的影响
            name=f"cross_{fc_i.name}_{fc_j.name}"
        )

        # 将特征对组合成单一索引并查找权重
        combined_index = feat_i * fc_j.vocab_size + feat_j
        cross_weight = cross_embedding(combined_index)  # 查表得到这对特征的权重
        cross_weights.append(cross_weight)

# 所有交叉特征权重相加
cross_logits = tf.add_n(cross_weights)

# 1. 特征嵌入：将稀疏的类别特征转换为稠密向量
group_feature_dict = {}
for group_name, _ in group_embedding_feature_dict.items():
    group_feature_dict[group_name] = concat_group_embedding(
        group_embedding_feature_dict, group_name, axis=1, flatten=True
    )  # B x (N * D) - 拼接所有特征的嵌入向量

# 2. 深度神经网络：逐层学习特征的非线性组合
deep_logits = []
for group_name, group_feature in group_feature_dict.items():
    # 构建多层神经网络
    deep_out = DNNs(
        units=dnn_units,  # 例如 [64, 32]
        activation="relu",  # ReLU激活函数
        dropout_rate=dnn_dropout_rate
    )(group_feature)

    # 输出层：将深度特征映射为预测分数
    deep_logit = tf.keras.layers.Dense(1, activation=None)(deep_out)
    deep_logits.append(deep_logit)

# Wide部分：线性特征 + 交叉特征
linear_logit = get_linear_logits(input_layer_dict, feature_columns)
cross_logit = get_cross_logits(input_layer_dict, feature_columns)

# Deep部分：多个特征组的深度网络输出
deep_logits = []
for group_name, group_feature in group_feature_dict.items():
    deep_out = DNNs(units=dnn_units, activation="relu", dropout_rate=dnn_dropout_rate)(
        group_feature
    )
    deep_logit = tf.keras.layers.Dense(1, activation=None)(deep_out)
    deep_logits.append(deep_logit)

# 联合训练：将Wide和Deep的输出相加
wide_deep_logits = add_tensor_func(deep_logits + [linear_logit, cross_logit])

# 最终预测：通过sigmoid函数输出点击概率
output = tf.keras.layers.Dense(1, activation="sigmoid")(wide_deep_logits)

# FM层的核心计算：0.5 * ((sum(v))^2 - sum(v^2))
# inputs: [batch_size, field_num, embedding_size]

# 先求和再平方：(∑v_i)^2
square_of_sum = tf.square(
    tf.reduce_sum(inputs, axis=1, keepdims=True)
)  # [B, 1, D]

# 先平方再求和：∑(v_i^2)
sum_of_square = tf.reduce_sum(
    inputs * inputs, axis=1, keepdims=True
)  # [B, 1, D]

# FM二阶交互项
cross_term = 0.5 * tf.reduce_sum(
    square_of_sum - sum_of_square, axis=2
)  # [B, 1]

# 1. 计算所有特征对的元素积交互
# group_pairwise: [batch_size, num_pairs, embedding_dim]
group_pairwise = pairwise_feature_interactions(
    group_feature, drop_rate=dropout_rate
)

# 2. 注意力权重计算：h^T · ReLU(W · (v_i ⊙ v_j) + b)
weighted_inputs = tf.matmul(
    group_pairwise, attention_weight
) + attention_bias  # [B, num_pairs, attention_factor]

activation = tf.nn.relu(weighted_inputs)
projected = tf.matmul(activation, attention_projection)  # [B, num_pairs, 1]

# 3. Softmax归一化得到注意力权重
attention_weights = tf.nn.softmax(projected, axis=1)

# 4. 加权求和：∑ a_ij · (v_i ⊙ v_j)
attention_output = tf.reduce_sum(
    tf.multiply(group_pairwise, attention_weights), axis=1
)  # [B, D]

# DIN注意力层的核心计算
# query: 候选广告 [batch_size, 1, embedding_dim]
# keys: 历史行为序列 [batch_size, seq_len, embedding_dim]

query = tf.squeeze(query, axis=1)  # [B, H]
length = tf.shape(keys)[-2]
query = tf.expand_dims(query, axis=1)  # [B, 1, H]

# 构建多角度交互特征：query, keys, query-keys, query*keys
att_inputs = tf.concat([
    tf.tile(query, [1, length, 1]),  # 重复query以匹配序列长度
    keys,                             # 历史行为
    query - keys,                     # 差异特征
    query * keys                      # 元素积特征
], axis=-1)  # [B, L, 4*H]

# 通过前馈网络计算注意力分数
hidden_layer = ffn_layer(att_inputs)  # [B, L, hidden_units]
scores = tf.keras.layers.Dense(1)(hidden_layer)  # [B, L, 1]

# 应用mask并进行加权求和（注意：不使用softmax归一化）
attention_weights = scores * mask  # [B, L, 1]
user_interest = tf.reduce_sum(keys * attention_weights, axis=1)  # [B, H]

# 兴趣提取层的辅助损失计算
# interest_states: [batch_size, seq_len, hidden_units]
# pos_behaviors: [batch_size, seq_len, embedding_dim] 正样本行为
# neg_behaviors: [batch_size, seq_len, embedding_dim] 负样本行为

# 用t时刻的兴趣预测t+1时刻的行为
current_interests = interest_states[:, :-1, :]      # [B, T-1, H]
next_pos_behaviors = pos_behaviors[:, 1:, :]        # [B, T-1, D]
next_neg_behaviors = neg_behaviors[:, 1:, :]        # [B, T-1, D]

# 拼接兴趣和行为，送入MLP预测
pos_input = tf.concat([current_interests, next_pos_behaviors], axis=-1)
neg_input = tf.concat([current_interests, next_neg_behaviors], axis=-1)

# 预测正负样本的概率
pos_probs = auxiliary_mlp(pos_input)  # [B, T-1, 1]
neg_probs = auxiliary_mlp(neg_input)  # [B, T-1, 1]

# 二元交叉熵损失
aux_loss = -tf.reduce_mean(
    tf.math.log(pos_probs + 1e-8) + tf.math.log(1 - neg_probs + 1e-8)
)

# AUGRU的前向传播
# interest_states: [batch_size, seq_len, hidden_units]
# target_item_embedding: [batch_size, embedding_dim]

# 1. 计算双线性注意力分数
# h_t * W * e_a
h_W = tf.tensordot(interest_states, bilinear_weight, axes=[[2], [0]])
target_expanded = tf.expand_dims(target_item_embedding, axis=1)
attention_scores = tf.reduce_sum(h_W * target_expanded, axis=2)  # [B, T]
attention_scores = tf.nn.softmax(attention_scores, axis=1)  # [B, T]

# 2. 逐步处理序列
hidden_state = tf.zeros([batch_size, hidden_units])
for t in range(seq_len):
    current_input = interest_states[:, t, :]     # [B, H]
    current_attention = attention_scores[:, t]   # [B]

    # 标准GRU计算
    update_gate = tf.nn.sigmoid(
        dense_input_update(current_input) + dense_hidden_update(hidden_state)
    )  # [B, H]

    reset_gate = tf.nn.sigmoid(
        dense_input_reset(current_input) + dense_hidden_reset(hidden_state)
    )  # [B, H]

    candidate_state = tf.nn.tanh(
        dense_input_candidate(current_input) +
        dense_hidden_candidate(reset_gate * hidden_state)
    )  # [B, H]

    # 关键：用注意力分数缩放更新门
    attention_expanded = tf.expand_dims(current_attention, axis=1)
    attention_expanded = tf.tile(attention_expanded, [1, hidden_units])
    attention_update_gate = attention_expanded * update_gate  # [B, H]

    # 更新隐藏状态
    hidden_state = (1 - attention_update_gate) * hidden_state + \
                   attention_update_gate * candidate_state

def build_shared_bottom_model(
        feature_columns,
        task_name_list,
        share_dnn_units=[128, 64],
        task_tower_dnn_units=[128, 64],
        ):
    # 输入层:将原始特征映射为 Keras 输入
    input_layer_dict = build_input_layer(feature_columns)

    # 嵌入层:为各特征组创建嵌入表,得到组内嵌入向量
    embedding_table_dict = build_group_feature_embedding_table_dict(
        feature_columns, input_layer_dict, prefix="embedding/"
    )

    # 合并嵌入:将多组嵌入拼接为共享 DNN 的输入
    dnn_inputs = concat_group_embedding(embedding_table_dict, 'dnn')

    # 共享底座:所有任务共享的特征抽取网络(Shared Bottom)
    shared_feature = DNNs(share_dnn_units)(dnn_inputs)

    # 任务塔:在共享特征上为每个任务构建独立塔并输出概率
    task_outputs = []
    for task_name in task_name_list:
        task_logit = DNNs(task_tower_dnn_units + [1])(shared_feature)
        task_prob = PredictLayer(name=f"task_{task_name}")(task_logit)
        task_outputs.append(task_prob)

    # 构建多任务模型:共享底座 + 多任务塔输出
    model = tf.keras.Model(inputs=list(input_layer_dict.values()), outputs=task_outputs)
    return model

from funrec import run_experiment

run_experiment('shared_bottom')

+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+
|   auc_is_click |   auc_is_like |   auc_long_view |   auc_macro |   gauc_is_click |   gauc_is_like |   gauc_long_view |   gauc_macro |   val_user_is_click |   val_user_is_like |   val_user_long_view |
+================+===============+=================+=============+=================+================+==================+==============+=====================+====================+======================+
|           0.61 |         0.431 |          0.4364 |      0.4925 |          0.5727 |         0.4433 |           0.4438 |       0.4866 |                 928 |                530 |                  925 |
+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+

def build_mmoe_model(
        feature_columns,
        task_name_list,
        expert_nums=4,
        expert_dnn_units=[128, 64],
        gate_dnn_units=[128, 64],
        task_tower_dnn_units=[128, 64],
        ):
    # 输入层:原始特征 → Keras 输入
    input_layer_dict = build_input_layer(feature_columns)

    # 嵌入层:为各特征组创建嵌入表,得到组内嵌入向量
    embedding_table_dict = build_group_feature_embedding_table_dict(
        feature_columns, input_layer_dict, prefix="embedding/"
    )

    # 合并嵌入:拼接为专家与门控的共同输入
    dnn_inputs = concat_group_embedding(embedding_table_dict, 'dnn')

    # 共享专家:多个并行 DNN(专家)供所有任务共享
    expert_outputs = [DNNs(expert_dnn_units, name=f"expert_{i}")(dnn_inputs)
                      for i in range(expert_nums)]
    # 形状 [B, E, D]:按专家维度堆叠,便于后续加权求和
    experts = tf.keras.layers.Lambda(lambda xs: tf.stack(xs, axis=1))(expert_outputs)

    # 任务门控:每个任务产生 softmax 权重,对专家加权融合
    task_features = []
    for idx, task_name in enumerate(task_name_list):
        gate_hidden = DNNs(gate_dnn_units, name=f"task_{idx}_gate_mlp")(dnn_inputs)
        gate_weights = tf.keras.layers.Dense(expert_nums, use_bias=False,
                                             activation='softmax',
                                             name=f"task_{idx}_gate_softmax")(gate_hidden)  # [B, E]
        # 加权融合:einsum('be,bed->bd') == sum_e w_e * expert_e
        task_mix = tf.keras.layers.Lambda(lambda x: tf.einsum('be,bed->bd', x[0], x[1]))([gate_weights, experts])
        task_features.append(task_mix)

    # 任务塔:基于融合特征为每个任务建塔并输出概率
    task_outputs = []
    for task_name, task_feat in zip(task_name_list, task_features):
        task_logit = DNNs(task_tower_dnn_units + [1])(task_feat)
        task_prob = PredictLayer(name=f"task_{task_name}")(task_logit)
        task_outputs.append(task_prob)

    # 构建模型:共享专家 + 任务门控 + 任务塔
    model = tf.keras.Model(inputs=list(input_layer_dict.values()), outputs=task_outputs)
    return model

run_experiment('mmoe')

+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+
|   auc_is_click |   auc_is_like |   auc_long_view |   auc_macro |   gauc_is_click |   gauc_is_like |   gauc_long_view |   gauc_macro |   val_user_is_click |   val_user_is_like |   val_user_long_view |
+================+===============+=================+=============+=================+================+==================+==============+=====================+====================+======================+
|         0.6036 |        0.4203 |          0.4586 |      0.4942 |          0.5728 |         0.4362 |           0.4759 |        0.495 |                 928 |                530 |                  925 |
+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+

def cgc_net(
        input_list,
        task_num,
        task_expert_num,
        shared_expert_num,
        task_expert_dnn_units,
        shared_expert_dnn_units,
        task_gate_dnn_units,
        shared_gate_dnn_units,
        leval_name=None,
        is_last=False):
    """CGC(共享专家 + 任务门控)核心结构(简化版)
    - 每个任务:拥有若干 Task-Experts;
    - 全局:拥有若干 Shared-Experts;
    - 每个任务 Gate 产生 softmax 权重,对其 Task-Experts 与 Shared-Experts 加权融合;
    - 若非最后一层:再用 Shared-Gate 融合所有任务的 Task-Experts 与 Shared-Experts,供下一层共享使用。
    input_list:为方便处理,给每个任务复制一份相同输入,最后一个为共享输入。
    """

    # 任务专家:每个任务创建 task_expert_num 个专家
    task_expert_list = []
    for i in range(task_num):
        task_expert_list.append([
            DNNs(task_expert_dnn_units, name=f"{leval_name}_task_{i}_expert_{j}")(input_list[i])
            for j in range(task_expert_num)
        ])

    # 共享专家:创建 shared_expert_num 个专家(共享输入使用 input_list[-1])
    shared_expert_list = [
        DNNs(shared_expert_dnn_units, name=f"{leval_name}_shared_expert_{i}")(input_list[-1])
        for i in range(shared_expert_num)
    ]

    # 任务门控与融合:对当前任务的(Task + Shared)专家集合进行 softmax 加权求和
    cgc_outputs = []
    fusion_expert_num = task_expert_num + shared_expert_num
    for i in range(task_num):
        cur_experts = task_expert_list[i] + shared_expert_list
        experts = tf.keras.layers.Lambda(lambda xs: tf.stack(xs, axis=1))(cur_experts)  # [B, E, D]

        gate_hidden = DNNs(task_gate_dnn_units, name=f"{leval_name}_task_{i}_gate")(input_list[i])
        gate_weights = tf.keras.layers.Dense(fusion_expert_num, use_bias=False, activation='softmax')(gate_hidden)  # [B, E]

        # 加权融合:einsum('be,bed->bd') == sum_e w_e * expert_e
        fused = tf.keras.layers.Lambda(lambda x: tf.einsum('be,bed->bd', x[0], x[1]))([gate_weights, experts])
        cgc_outputs.append(fused)

    # 若非最后一层:共享门控融合所有任务专家与共享专家,作为下一层共享输入
    if not is_last:
        # 展平所有任务的专家 + 共享专家
        all_task_experts = [e for task in task_expert_list for e in task]
        cur_experts = all_task_experts + shared_expert_list
        experts_all = tf.keras.layers.Lambda(lambda xs: tf.stack(xs, axis=1))(cur_experts)  # [B, E_all, D]
        cur_expert_num = len(cur_experts)

        shared_gate_hidden = DNNs(shared_gate_dnn_units, name=f"{leval_name}_shared_gate")(input_list[-1])
        shared_gate_weights = tf.keras.layers.Dense(cur_expert_num, use_bias=False, activation='softmax')(shared_gate_hidden)  # [B, E_all]
        shared_fused = tf.keras.layers.Lambda(lambda x: tf.einsum('be,bed->bd', x[0], x[1]))([shared_gate_weights, experts_all])
        cgc_outputs.append(shared_fused)

    return cgc_outputs

def build_ple_model(
        feature_columns,
        task_name_list,
        ple_level_nums=1,
        task_expert_num=4,
        shared_expert_num=2,
        task_expert_dnn_units=[128, 64],
        shared_expert_dnn_units=[128, 64],
        task_gate_dnn_units=[128, 64],
        shared_gate_dnn_units=[128, 64],
        task_tower_dnn_units=[128, 64],
        ):
    # 1) 输入与嵌入:构建输入层/分组嵌入,拼接为 PLE 的共享输入
    input_layer_dict = build_input_layer(feature_columns)
    group_embedding_feature_dict = build_group_feature_embedding_table_dict(
        feature_columns, input_layer_dict, prefix="embedding/"
    )
    dnn_inputs = concat_group_embedding(group_embedding_feature_dict, 'dnn')

    # 2) 级联 PLE(CGC)层:每层包含"任务专家 + 共享专家 + 门控",最后一层仅输出任务特征
    task_num = len(task_name_list)
    ple_input_list = [dnn_inputs] * (task_num + 1)  # 前 task_num 为各任务输入,末尾为共享输入
    for i in range(ple_level_nums):
        is_last = (i == ple_level_nums - 1)
        ple_input_list = cgc_net(
            ple_input_list,
            task_num,
            task_expert_num,
            shared_expert_num,
            task_expert_dnn_units,
            shared_expert_dnn_units,
            task_gate_dnn_units,
            shared_gate_dnn_units,
            leval_name=f"cgc_level_{i}",
            is_last=is_last
        )

    # 3) 任务塔与输出:将各任务特征送入塔 DNN,得到每个任务的概率输出
    task_output_list = []
    for i in range(task_num):
        task_logit = DNNs(task_tower_dnn_units + [1])(ple_input_list[i])
        task_prob = PredictLayer(name="task_" + task_name_list[i])(task_logit)
        task_output_list.append(task_prob)

    # 4) 构建模型:输入为所有原始输入层,输出为各任务概率
    model = tf.keras.Model(inputs=list(input_layer_dict.values()), outputs=task_output_list)
    return model

run_experiment('ple')

+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+
|   auc_is_click |   auc_is_like |   auc_long_view |   auc_macro |   gauc_is_click |   gauc_is_like |   gauc_long_view |   gauc_macro |   val_user_is_click |   val_user_is_like |   val_user_long_view |
+================+===============+=================+=============+=================+================+==================+==============+=====================+====================+======================+
|         0.5975 |        0.4191 |          0.4474 |       0.488 |           0.574 |         0.4434 |           0.4561 |       0.4912 |                 928 |                530 |                  925 |
+----------------+---------------+-----------------+-------------+-----------------+----------------+------------------+--------------+---------------------+--------------------+----------------------+