- 结构
TransformerLM
不会初始化其成员变量。为什么这会起作用?
- 结构
TransformerLM
没有初始化程序或默认值,为什么可以 let model = TransformerLM(contentsOfPythonCheckpointFile: checkpoint, config: config, scope: "model")
工作?
struct TransformerLM {
var embedding: Embedding
var positionalEmbeddings: Tensor<Float>
var layers: [EncoderLayer]
var norm: LayerNorm<Float>
func call(_ tokens: Tensor<Int32>, states: inout [AttentionContext]) -> Tensor<Float> {
let positions = (0..<tokens.shape[1]).map { Int32($0 + states[0].key.shape[1]) }
let positionsTensor = Tensor<Int32>(shape: [1, tokens.shape[1]], scalars: positions)
var h = embedding(tokens)
h = h + positionalEmbeddings.gathering(atIndices: positionsTensor)
for i in 0..<layers.count {
h = layers[i](h, state: &states[i])
}
h = norm(h)
let tmp = TimeDistributed(
Dense(weight: embedding.weight.transposed(), bias: Tensor(0.0), activation: identity))
let logits = tmp(h) // a somewhat hacky way to share weights
return logits
}
git:定义
调用