Аренда GPU для NVIDIA Triton Inference Server
запуск за 5 минут,
от 168 ₽/час

# 1. Подготовить model repository (см. вкладку Model repo)
mkdir -p /opt/triton/models

# 2. Запустить Triton (gRPC 8001, HTTP 8000, metrics 8002)
docker run --gpus all --rm -d \
  --name triton \
  --shm-size=2g --ulimit memlock=-1 --ulimit stack=67108864 \
  -p 8000:8000 -p 8001:8001 -p 8002:8002 \
  -v /opt/triton/models:/models \
  nvcr.io/nvidia/tritonserver:24.10-py3 \
  tritonserver --model-repository=/models \
    --strict-model-config=false \
    --log-verbose=1

# 3. Проверить health
curl -v http://localhost:8000/v2/health/ready
curl http://localhost:8002/metrics | head

# Структура model repository:
/opt/triton/models/
├── resnet50_onnx/
│   ├── 1/
│   │   └── model.onnx
│   └── config.pbtxt
└── llama3_trtllm/
    ├── 1/
    │   └── rank0.engine
    └── config.pbtxt

# Пример config.pbtxt для ONNX-модели с dynamic batching:
cat > /opt/triton/models/resnet50_onnx/config.pbtxt <<'EOF'
name: "resnet50_onnx"
platform: "onnxruntime_onnx"
max_batch_size: 32
input  [{ name: "input"  data_type: TYPE_FP32 dims: [3, 224, 224] }]
output [{ name: "output" data_type: TYPE_FP32 dims: [1000] }]
instance_group [{ count: 2 kind: KIND_GPU }]
dynamic_batching {
  preferred_batch_size: [4, 8, 16]
  max_queue_delay_microseconds: 100
}
EOF

# /opt/triton/models/embed_e5/
# ├── 1/
# │   └── model.py
# └── config.pbtxt

# config.pbtxt:
name: "embed_e5"
backend: "python"
max_batch_size: 16
input  [{ name: "text"      data_type: TYPE_STRING dims: [1] }]
output [{ name: "embedding" data_type: TYPE_FP32   dims: [1024] }]
instance_group [{ count: 1 kind: KIND_GPU }]

# 1/model.py:
import numpy as np, triton_python_backend_utils as pb_utils
from sentence_transformers import SentenceTransformer

class TritonPythonModel:
    def initialize(self, args):
        self.model = SentenceTransformer(
            'intfloat/multilingual-e5-large', device='cuda'
        )

    def execute(self, requests):
        responses = []
        for req in requests:
            text = pb_utils.get_input_tensor_by_name(req, 'text')
            texts = [b.decode() for b in text.as_numpy().flatten()]
            emb = self.model.encode(texts, normalize_embeddings=True)
            out = pb_utils.Tensor('embedding', emb.astype(np.float32))
            responses.append(pb_utils.InferenceResponse([out]))
        return responses

pip install tritonclient[grpc] numpy

python3 - <<'PY'
import numpy as np
import tritonclient.grpc as grpc

client = grpc.InferenceServerClient(url='localhost:8001')
assert client.is_server_ready()

img = np.random.rand(1, 3, 224, 224).astype(np.float32)
inp = grpc.InferInput('input', img.shape, 'FP32')
inp.set_data_from_numpy(img)

out = client.infer(model_name='resnet50_onnx', inputs=[inp])
print(out.as_numpy('output').shape)
PY