# Gateway LoadBalancer IP (provisioned by MetalLB)
GATEWAY_IP=$(kubectl get svc -n envoy-gateway-system \
  -l gateway.envoyproxy.io/owning-gateway-name \
  -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')

# LiteLLM hostname
LITELLM_DOMAIN="litellm.$(grep base_domain_name env/local/global_config.yaml | awk '{print $2}' | tr -d '\"')"

# Get the master key (or use a virtual key created via LiteLLM UI/API)
TOKEN=$(kubectl get secret litellm-master-key -n litellm \
  -o jsonpath='{.data.master_key}' | base64 -d)

# Keycloak hostname (from the Keycloak CR)
KEYCLOAK_DOMAIN=$(kubectl get keycloaks -n keycloak \
  -o jsonpath='{.items[0].spec.hostname.hostname}')

# Inference gateway hostname: keycloak.<base> → inference.<base>
GATEWAY_DOMAIN="inference.${KEYCLOAK_DOMAIN#keycloak.}"

# OIDC client credentials (created by the keycloak_config role)
REALM="inference"
CLIENT_ID=$(kubectl get secret keycloak-client-secret -n keycloak \
  -o jsonpath='{.data.client-id}' | base64 -d)
CLIENT_SECRET=$(kubectl get secret keycloak-client-secret -n keycloak \
  -o jsonpath='{.data.client-secret}' | base64 -d)

# Get a JWT (client-credentials flow — service-to-service)
TOKEN=$(curl -sk --noproxy '*' --resolve ${KEYCLOAK_DOMAIN}:443:${GATEWAY_IP} \
  https://${KEYCLOAK_DOMAIN}/realms/${REALM}/protocol/openid-connect/token \
  -d "grant_type=client_credentials" \
  -d "client_id=${CLIENT_ID}" \
  -d "client_secret=${CLIENT_SECRET}" \
  | python3 -c "import json,sys; print(json.load(sys.stdin)['access_token'])")

curl -sk --noproxy '*' --resolve ${LITELLM_DOMAIN}:443:${GATEWAY_IP} \
  -H "Authorization: Bearer ${TOKEN}" \
  -H "Content-Type: application/json" \
  https://${LITELLM_DOMAIN}/v1/chat/completions \
  -d '{
    "model": "llama-3-2-1b",
    "messages": [{"role":"user","content":"Hello!"}],
    "max_tokens": 50
  }'

curl -sk --noproxy '*' --resolve ${GATEWAY_DOMAIN}:443:${GATEWAY_IP} \
  -H "Authorization: Bearer ${TOKEN}" \
  -H "Content-Type: application/json" \
  https://${GATEWAY_DOMAIN}/llm-inference/llama-3-2-1b/v1/chat/completions \
  -d '{
    "model": "llama-3-2-1b",
    "messages": [{"role":"user","content":"Hello!"}],
    "max_tokens": 50
  }'

import os
from openai import OpenAI

# auth_provider=litellm:
client = OpenAI(
    base_url=f"https://{os.environ['LITELLM_DOMAIN']}/v1",
    api_key=os.environ["TOKEN"],   # LiteLLM virtual key or master key
    http_client=__import__("httpx").Client(verify=False),  # for self-signed certs
)

# auth_provider=keycloak:
# client = OpenAI(
#     base_url=f"https://{os.environ['GATEWAY_DOMAIN']}/llm-inference/llama-3-2-1b/v1",
#     api_key=os.environ["TOKEN"],   # Keycloak JWT
#     http_client=__import__("httpx").Client(verify=False),
# )

resp = client.chat.completions.create(
    model="llama-3-2-1b",
    messages=[{"role": "user", "content": "Hello!"}],
)
print(resp.choices[0].message.content)

import os
from langchain_openai import ChatOpenAI

llm = ChatOpenAI(
    base_url=os.environ['BASE_URL'],
    api_key=os.environ['TOKEN'],
    model="llama-3-2-1b",
)
# Use any LangChain chain / retriever / vectorstore on top.

import os
from llama_index.llms.openai_like import OpenAILike

llm = OpenAILike(
    api_base=os.environ['BASE_URL'],
    api_key=os.environ['TOKEN'],
    model="llama-3-2-1b",
    is_chat_model=True,
)

import os
from langgraph.prebuilt import create_react_agent
from langchain_openai import ChatOpenAI

llm = ChatOpenAI(
    base_url=os.environ['BASE_URL'],
    api_key=os.environ['TOKEN'],
    model="llama-3-2-1b",
)
agent = create_react_agent(llm, tools=[...])

import os
from crewai import LLM, Agent

llm = LLM(
    model="openai/llama-3-2-1b",
    base_url=os.environ['BASE_URL'],
    api_key=os.environ['TOKEN'],
)
agent = Agent(role="Researcher", llm=llm, ...)

import os
from openai import AsyncOpenAI
from agents import Agent, OpenAIChatCompletionsModel

client = AsyncOpenAI(
    base_url=os.environ['BASE_URL'],
    api_key=os.environ['TOKEN'],
)
agent = Agent(
    name="MyAgent",
    model=OpenAIChatCompletionsModel("llama-3-2-1b", openai_client=client),
)

Base URL: https://litellm.inference-example.com/v1  (litellm mode)
      or: https://inference.inference-example.com/llm-inference/<model>/v1  (keycloak mode)
API Key:  

./model-manager deploy <model-name>

./es_auto_installer.sh install inference_services --env <name>

# my-sklearn-model.yaml
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: iris-classifier
  namespace: llm-inference         # or any KServe-enabled namespace
spec:
  predictor:
    sklearn:
      storageUri: "pvc://shared-models/iris/"   # or s3://, gs://, http://, oci://
      resources:
        requests:
          cpu: "200m"
          memory: "512Mi"

kubectl apply -f my-sklearn-model.yaml
kubectl get inferenceservice -n llm-inference iris-classifier -w

curl -sk --noproxy '*' --resolve ${GATEWAY_DOMAIN}:443:${GATEWAY_IP} \
  -H "Authorization: Bearer ${TOKEN}" \
  -H "Content-Type: application/json" \
  https://${GATEWAY_DOMAIN}/llm-inference/iris-classifier/v2/models/iris-classifier/infer \
  -d '{
    "inputs": [{
      "name": "input-0",
      "shape": [1, 4],
      "datatype": "FP32",
      "data": [5.1, 3.5, 1.4, 0.2]
    }]
  }'

import os, requests

URL = (f"https://{os.environ['GATEWAY_DOMAIN']}"
       "/llm-inference/iris-classifier/v2/models/iris-classifier/infer")

resp = requests.post(
    URL,
    headers={"Authorization": f"Bearer {os.environ['TOKEN']}"},
    json={
        "inputs": [{
            "name": "input-0",
            "shape": [1, 4],
            "datatype": "FP32",
            "data": [5.1, 3.5, 1.4, 0.2],
        }]
    },
    verify=False,   # selfsigned TLS in default install
)
print(resp.json())

spec:
  predictor:
    containers:
      - name: kserve-container
        image: myregistry/my-custom-server:latest
        ports:
          - containerPort: 8080
            protocol: TCP

listeners:
  - name: https
    protocol: HTTPS
    port: 443
    hostname: "*.<base_domain_name>"   # accepts any subdomain

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: tenant-a
  namespace: tenant-a
spec:
  parentRefs:
    - name: eg-gateway
      namespace: envoy-gateway-system
  hostnames:
    - tenant-a.<base>
  rules:
    - backendRefs:
        - name: tenant-a-svc
          port: 8080

spec:
  hostnames: [api.<base>]
  rules:
    - matches:
        - path: { type: PathPrefix, value: /tenant-a }
      backendRefs:
        - { name: tenant-a-svc, port: 8080 }
    - matches:
        - path: { type: PathPrefix, value: /tenant-b }
      backendRefs:
        - { name: tenant-b-svc, port: 8080 }

rules:
  - matches: [{ path: { type: PathPrefix, value: /v1 } }]
    backendRefs: [{ name: api-v1, port: 8080 }]
  - matches: [{ path: { type: PathPrefix, value: /v2 } }]
    backendRefs: [{ name: api-v2, port: 8080 }]

rules:
  - matches: [{ path: { type: PathPrefix, value: /v1 } }]
    backendRefs:
      - { name: api-stable, port: 8080, weight: 90 }
      - { name: api-canary, port: 8080, weight: 10 }

rules:
  - matches:
      - path: { type: PathPrefix, value: /v1 }
        headers:
          - { name: x-experiment, value: variant-b }
    backendRefs: [{ name: api-variant-b, port: 8080 }]
  - matches: [{ path: { type: PathPrefix, value: /v1 } }]
    backendRefs: [{ name: api-stable, port: 8080 }]

rules:
  - matches: [{ path: { type: PathPrefix, value: /api }, method: GET }]
    backendRefs: [{ name: api-readonly-cache, port: 8080 }]
  - matches: [{ path: { type: PathPrefix, value: /api }, method: POST }]
    backendRefs: [{ name: api-write, port: 8080 }]

apiVersion: gateway.networking.k8s.io/v1beta1
kind: ReferenceGrant
metadata:
  name: allow-from-gateway
  namespace: my-app-ns
spec:
  from:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      namespace: envoy-gateway-system
  to:
    - group: ""
      kind: Service

apiVersion: gateway.envoyproxy.io/v1alpha1
kind: BackendTrafficPolicy
metadata:
  name: tenant-a-ratelimit
spec:
  targetRefs:
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      name: tenant-a
  rateLimit:
    type: Local
    local:
      rules:
        - limit: { requests: 100, unit: Minute }