text-generation-inference

Updated 6 hours ago

Versions

Browse available versions organized by version lines.

Line latest

2 versions·End of life date is unavailable

Updated

version

Pull URL

vulnerabilities

exploitability

Compressed size

Updated today

3.3.7

Vulnerabilities

Exploitability-

Compressed size6396.73 MB

3.3.7 dev

Vulnerabilities

Exploitability-

Compressed size6397.75 MB

text-generation-inference 3.3.7

Updated 6 hours ago·

6396.73 MB / 275.96 MB

Vulnerabilities

Tags

User

root

Ports

Environment variables

CUDA_VERSION=12.4.0EXLLAMA_NO_FLASH_ATTN=1HF_HOME=/dataHF_HUB_ENABLE_HF_TRANSFER=1HF_KERNELS_CACHE=/kernelsLD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64NVARCH=x86_64NVIDIA_DRIVER_CAPABILITIES=compute,utility

NVIDIA_REQUIRE_CUDA=cuda>=12.4 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526 brand=tesla,driver>=535,driver<536 brand=unknown,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=geforce,driver>=535,driver<536 brand=geforcertx,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=titan,driver>=535,driver<536 brand=titanrtx,driver>=535,driver<536

NVIDIA_VISIBLE_DEVICES=allNV_CUDA_COMPAT_PACKAGE=cuda-compat-12-4NV_CUDA_CUDART_VERSION=12.4.99-1

PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/src/.venv/bin/

PORT=80PYTHON_VERSION=3.11SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crtVIRTUAL_ENV=/usr/src/.venv/

Entrypoint

/tgi-entrypoint.sh

CMD

Volumes

Working directory

/usr/src

Stop signal

Configuration

12345678910111213141516171819202122232425

{
  "Env": [
    "CUDA_VERSION=12.4.0",
    "EXLLAMA_NO_FLASH_ATTN=1",
    "HF_HOME=/data",
    "HF_HUB_ENABLE_HF_TRANSFER=1",
    "HF_KERNELS_CACHE=/kernels",
    "LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64",
    "NVARCH=x86_64",
    "NVIDIA_DRIVER_CAPABILITIES=compute,utility",
    "NVIDIA_REQUIRE_CUDA=cuda>=12.4 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=525,driver<526 brand=unknown,driver>=525,driver<526 brand=nvidia,driver>=525,driver<526 brand=nvidiartx,driver>=525,driver<526 brand=geforce,driver>=525,driver<526 brand=geforcertx,driver>=525,driver<526 brand=quadro,driver>=525,driver<526 brand=quadrortx,driver>=525,driver<526 brand=titan,driver>=525,driver<526 brand=titanrtx,driver>=525,driver<526 brand=tesla,driver>=535,driver<536 brand=unknown,driver>=535,driver<536 brand=nvidia,driver>=535,driver<536 brand=nvidiartx,driver>=535,driver<536 brand=geforce,driver>=535,driver<536 brand=geforcertx,driver>=535,driver<536 brand=quadro,driver>=535,driver<536 brand=quadrortx,driver>=535,driver<536 brand=titan,driver>=535,driver<536 brand=titanrtx,driver>=535,driver<536",
    "NVIDIA_VISIBLE_DEVICES=all",
    "NV_CUDA_COMPAT_PACKAGE=cuda-compat-12-4",
    "NV_CUDA_CUDART_VERSION=12.4.99-1",
    "PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/src/.venv/bin/",
    "PORT=80",
    "PYTHON_VERSION=3.11",
    "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt",
    "VIRTUAL_ENV=/usr/src/.venv/"
  ],
  "Entrypoint": [
    "/tgi-entrypoint.sh"
  ],
  "WorkingDir": "/usr/src"
}