0
  1. 结构TransformerLM不会初始化其成员变量。为什么这会起作用?
  2. 结构TransformerLM没有初始化程序或默认值,为什么可以 let model = TransformerLM(contentsOfPythonCheckpointFile: checkpoint, config: config, scope: "model")工作?
struct TransformerLM {
    var embedding: Embedding
    var positionalEmbeddings: Tensor<Float>
    var layers: [EncoderLayer]
    var norm: LayerNorm<Float>

    func call(_ tokens: Tensor<Int32>, states: inout [AttentionContext]) -> Tensor<Float> {
        let positions = (0..<tokens.shape[1]).map { Int32($0 + states[0].key.shape[1]) }
        let positionsTensor = Tensor<Int32>(shape: [1, tokens.shape[1]], scalars: positions)
        var h = embedding(tokens)
        h = h + positionalEmbeddings.gathering(atIndices: positionsTensor)
        for i in 0..<layers.count {
            h = layers[i](h, state: &states[i])
        }
        h = norm(h)
        let tmp = TimeDistributed(
            Dense(weight: embedding.weight.transposed(), bias: Tensor(0.0), activation: identity))
        let logits = tmp(h) // a somewhat hacky way to share weights
        return logits
}

git:定义
调用

4

0 回答 0