Skip to content

Commit 9b908fb

Browse files
authored
Merge pull request #181 from VikParuchuri/main
Fix rope base issue with llama 3
2 parents 6253c6b + 8922741 commit 9b908fb

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

model.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def from_name(cls, name: str):
6565
"Mistral-7B": dict(n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=32000),
6666
"stories15M": dict(n_layer=6, n_head=6, dim=288),
6767
"stories110M": dict(n_layer=12, n_head=12, dim=768),
68-
"Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256),
68+
"Llama-3-8B": dict(block_size=8192, n_layer=32, n_head=32, n_local_heads=8, dim=4096, intermediate_size=14336, vocab_size=128256, rope_base=500000),
6969
}
7070

7171
class KVCache(nn.Module):

0 commit comments

Comments
 (0)