Skip to content

Nvidia

NVIDIA #

Bases: OpenAILike

NVIDIA's API Catalog Connector.

Source code in llama-index-integrations/llms/llama-index-llms-nvidia/llama_index/llms/nvidia/base.py 
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
class NVIDIA(OpenAILike):
    """NVIDIA's API Catalog Connector."""

    _mode: str = PrivateAttr("nvidia")

    def __init__(
        self,
        model: str = DEFAULT_MODEL,
        nvidia_api_key: Optional[str] = None,
        api_key: Optional[str] = None,
        max_tokens: Optional[int] = 1024,
        **kwargs: Any,
    ) -> None:
        api_key = get_from_param_or_env(
            "api_key",
            nvidia_api_key or api_key,
            "NVIDIA_API_KEY",
            "NO_API_KEY_PROVIDED",
        )

        super().__init__(
            model=model,
            api_key=api_key,
            api_base=BASE_URL,
            max_tokens=max_tokens,
            is_chat_model=True,
            default_headers={"User-Agent": "llama-index-llms-nvidia"},
            **kwargs,
        )

    @property
    def available_models(self) -> List[Model]:
        exclude = {
            "mistralai/mixtral-8x22b-v0.1",  # not a /chat/completion endpoint
        }
        # do not exclude models in nim mode. the nim administrator has control
        # over the model name and may deploy an excluded name on the nim's
        # /chat/completion endpoint.
        if self._mode == "nim":
            exclude = set()
        return [
            model
            for model in self._get_client().models.list().data
            if model.id not in exclude
        ]

    @classmethod
    def class_name(cls) -> str:
        return "NVIDIA"

    def mode(
        self,
        mode: Optional[Literal["nvidia", "nim"]] = "nvidia",
        *,
        base_url: Optional[str] = None,
        model: Optional[str] = None,
        api_key: Optional[str] = None,
    ) -> "NVIDIA":
        """
        Change the mode.

        There are two modes, "nvidia" and "nim". The "nvidia" mode is the default
        mode and is used to interact with hosted NIMs. The "nim" mode is used to
        interact with NVIDIA NIM endpoints, which are typically hosted on-premises.

        For the "nvidia" mode, the "api_key" parameter is available to specify
        your API key. If not specified, the NVIDIA_API_KEY environment variable
        will be used.

        For the "nim" mode, the "base_url" parameter is required and the "model"
        parameter may be necessary. Set base_url to the url of your local NIM
        endpoint. For instance, "https://localhost:9999/v1". Additionally, the
        "model" parameter must be set to the name of the model inside the NIM.
        """
        if mode == "nim":
            if not base_url:
                raise ValueError("base_url is required for nim mode")
        if mode == "nvidia":
            api_key = get_from_param_or_env(
                "api_key",
                api_key,
                "NVIDIA_API_KEY",
            )
            base_url = base_url or BASE_URL

        self._mode = mode
        if base_url:
            self.api_base = base_url
        if model:
            self.model = model
        if api_key:
            self.api_key = api_key

        return self

mode #

mode(mode: Optional[Literal['nvidia', 'nim']] = 'nvidia', *, base_url: Optional[str] = None, model: Optional[str] = None, api_key: Optional[str] = None) -> NVIDIA

Change the mode.

There are two modes, "nvidia" and "nim". The "nvidia" mode is the default mode and is used to interact with hosted NIMs. The "nim" mode is used to interact with NVIDIA NIM endpoints, which are typically hosted on-premises.

For the "nvidia" mode, the "api_key" parameter is available to specify your API key. If not specified, the NVIDIA_API_KEY environment variable will be used.

For the "nim" mode, the "base_url" parameter is required and the "model" parameter may be necessary. Set base_url to the url of your local NIM endpoint. For instance, "https://localhost:9999/v1". Additionally, the "model" parameter must be set to the name of the model inside the NIM.

Source code in llama-index-integrations/llms/llama-index-llms-nvidia/llama_index/llms/nvidia/base.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def mode(
    self,
    mode: Optional[Literal["nvidia", "nim"]] = "nvidia",
    *,
    base_url: Optional[str] = None,
    model: Optional[str] = None,
    api_key: Optional[str] = None,
) -> "NVIDIA":
    """
    Change the mode.

    There are two modes, "nvidia" and "nim". The "nvidia" mode is the default
    mode and is used to interact with hosted NIMs. The "nim" mode is used to
    interact with NVIDIA NIM endpoints, which are typically hosted on-premises.

    For the "nvidia" mode, the "api_key" parameter is available to specify
    your API key. If not specified, the NVIDIA_API_KEY environment variable
    will be used.

    For the "nim" mode, the "base_url" parameter is required and the "model"
    parameter may be necessary. Set base_url to the url of your local NIM
    endpoint. For instance, "https://localhost:9999/v1". Additionally, the
    "model" parameter must be set to the name of the model inside the NIM.
    """
    if mode == "nim":
        if not base_url:
            raise ValueError("base_url is required for nim mode")
    if mode == "nvidia":
        api_key = get_from_param_or_env(
            "api_key",
            api_key,
            "NVIDIA_API_KEY",
        )
        base_url = base_url or BASE_URL

    self._mode = mode
    if base_url:
        self.api_base = base_url
    if model:
        self.model = model
    if api_key:
        self.api_key = api_key

    return self