GLA Model Configuration (gla/configuration_gla.py)

Hugging Face configuration for Gated Linear Attention models. Reuse it when building or loading an FLA GLA checkpoint that needs architecture hyperparameters.

# Source: https://github.com/josStorer/RWKV-Runner | License: MIT | Copyright 2023 josStorer
# Provided via justclaw.it.com; keep this notice with the code.
# -*- coding: utf-8 -*-

from typing import Optional

from transformers.configuration_utils import PretrainedConfig


class GLAConfig(PretrainedConfig):

    model_type = 'gla'
    keys_to_ignore_at_inference = ['past_key_values']

    def __init__(
        self,
        vocab_size: int = 32000,
        hidden_size: int = 2048,
        expand_k: int = 0.5,
        expand_v: int = 1,
        hidden_ratio: Optional[int] = 4,
        intermediate_size: Optional[int] = None,
        num_hidden_layers: int = 24,
        num_heads: int = 4,
        num_kv_heads: Optional[int] = None,
        feature_map: Optional[str] = None,
        attn_mo

... (truncated -- full source via MCP)