Advertisement
pszemraj

reformer-crime and punishment-original config

Oct 6th, 2022
206
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
JSON 0.83 KB | None | 0 0
  1. config = {
  2.     "attention_head_size": 64,
  3.     "attn_layers": ["local", "lsh", "local", "lsh", "local", "lsh"],
  4.     "axial_pos_embds": True,
  5.     "sinusoidal_pos_embds": False,
  6.     "axial_pos_embds_dim": [64, 192],
  7.     "axial_pos_shape": [512, 1024],
  8.     "lsh_attn_chunk_length": 64,
  9.     "local_attn_chunk_length": 64,
  10.     "feed_forward_size": 512,
  11.     "hidden_act": "relu",
  12.     "hidden_size": 256,
  13.     "is_decoder": True,
  14.     "max_position_embeddings": 524288,
  15.     "num_attention_heads": 2,
  16.     "num_buckets": [64, 128],
  17.     "num_hashes": 1,
  18.     "vocab_size": 320,
  19.     "lsh_attention_probs_dropout_prob": 0.0,
  20.     "lsh_num_chunks_before": 1,
  21.     "lsh_num_chunks_after": 0,
  22.     "local_num_chunks_before": 1,
  23.     "local_num_chunks_after": 0,
  24.     "local_attention_probs_dropout_prob": 0.025,
  25.     "hidden_dropout_prob": 0.025,
  26. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement