rasa.nlu.tokenizers.mitie_tokenizer

MitieTokenizer Objects

@DefaultV1Recipe.register(
    DefaultV1Recipe.ComponentType.MESSAGE_TOKENIZER, is_trainable=False
)

class MitieTokenizer(Tokenizer):
    """Tokenizes messages using the `mitie` library."""

get_default_config

@staticmethod
def get_default_config() -> Dict[Text, Any]:
    """Returns default config (see parent class for full docstring)."""
    pass

required_packages

@staticmethod
def required_packages() -> List[Text]:
    """Any extra python dependencies required for this component to run."""
    pass

create

@classmethod
def create(cls, config: Dict[Text, Any], model_storage: ModelStorage,
               resource: Resource,
               execution_context: ExecutionContext) -> MitieTokenizer:
    """Creates a new component (see parent class for full docstring)."""
    pass

tokenize

def tokenize(message: Message, attribute: Text) -> List[Token]:
    """Tokenizes the text of the provided attribute of the incoming message."""
    pass