Furiosa RNGD NPU에서 LLM 모델 서빙하기

from llama_index.core import Document
from llama_index.core.node_parser import SentenceSplitter
from llama_index.vector_stores.qdrant import QdrantVectorStore

async def ingest_documents(
    documents: List[Document],
    collection_name: str,
    embed_model: BaseEmbedding,
    node_parser: SentenceSplitter,
    client: AsyncQdrantClient,
    translator: TranslationClient,
    target_language: Optional[str] = None,
):
    # Step 1: Collection 생성 (동적 벡터 크기 결정)
    dummy_embedding = embed_model.get_text_embedding("dummy text")
    vector_size = len(dummy_embedding)  # 1024 for BGE-large-en

    await client.create_collection(
        collection_name=collection_name,
        vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE),
    )

    vector_store = QdrantVectorStore(aclient=client, collection_name=collection_name)

    # Step 2: Document를 Node로 분할 (Chunking)
    nodes = node_parser.get_nodes_from_documents(documents, show_progress=True)

    # Step 3: 다국어 검색 최적화를 위한 번역 (선택적)
    if target_language:
        original_texts = [node.get_content() for node in nodes]
        translated_texts = await translator.translate(original_texts, target_language)

        for i, node in enumerate(nodes):
            node.set_content(translated_texts[i])
            node.metadata["original_text"] = original_texts[i]
            node.metadata["translated_to"] = target_language

    # Step 4: Embedding 생성
    for node in nodes:
        node.embedding = embed_model.get_text_embedding(node.get_content())

    # Step 5: Vector Store에 저장
    await vector_store.async_add(nodes)

from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_core.runnables import RunnablePassthrough, RunnableLambda

# RAG 프롬프트 템플릿
RAG_PROMPT = ChatPromptTemplate.from_messages([
    ("system", """You are a helpful and professional AI assistant.

**Your Core Mission:**
Your goal is to answer user questions based *only* on the provided 'Context'.

**Key Instructions:**
1. **Assume User's Intent:** Handle typos and misspellings gracefully.
2. **NEVER Correct the User:** Provide direct answers without pointing out mistakes.
3. **Strictly Context-Based:** All answers must come from the provided Context.
4. **Handle Missing Information:** Politely state when information is unavailable.

Context:
{context}"""),
    MessagesPlaceholder(variable_name="chat_history"),
    ("human", "{question}"),
])

# LCEL 체인 구성
def get_chain_for_request(request: ChatRequest) -> Runnable:
    configured_llm = llm.bind(
        temperature=request.temperature,
        stop=["<|eot_id|>"],  # Llama 3.1 EOS 토큰
    )

    retrieval_chain = (
        RunnableLambda(retrieve_documents)
        | RunnableLambda(_format_docs)
    )

    chain = (
        RunnableLambda(lambda messages: {"messages": messages})
        | RunnablePassthrough.assign(
            question=itemgetter("messages") | RunnableLambda(_get_last_user_query),
        )
        | RunnablePassthrough.assign(
            context=itemgetter("question") | retrieval_chain
        )
        | {
            "chat_history": itemgetter("messages")
                | RunnableLambda(_convert_to_langchain_messages)
                | RunnableLambda(lambda msgs: msgs[:-1]),  # 마지막 메시지 제외
            "question": itemgetter("question"),
            "context": itemgetter("context"),
        }
        | RAG_PROMPT
        | configured_llm
        | StrOutputParser()
    )

    return chain

# 초기 계획: LoRA Fine-tuning
from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=8,                    # Rank
    lora_alpha=16,          # Scaling factor
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.1,
)

def build_translation_request(req: TranslateRequest, text: str) -> dict:
    src_lang = LANGUAGE_LABELS.get(req.source, "")
    tgt_lang = LANGUAGE_LABELS.get(req.target, None)

    # System Prompt: 역할 정의 및 출력 형식 제어
    system_prompt = (
        f"You are an expert translator. "
        f"Your task is to accurately translate text from {src_lang} to {tgt_lang}. "
        f"Provide only the translated text, without any additional explanations or introductions."
    )

    # User Prompt: Llama Native Instruction Format 활용
    user_prompt = f"<s>[INST]Translate this from {src_lang} to {tgt_lang}:\n{src_lang}: {text}\n{tgt_lang}:[/INST]"

    return {
        "model": "furiosa-ai/Llama-3.1-8B-Instruct-FP8",
        "messages": [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ],
        "temperature": 0.3,  # 낮은 temperature로 일관성 확보
        "stream": False,
    }

# FuriosaLLM 서버 실행
FURIOSA_GENERATOR_WARMUP_ENABLE=1 furiosa-llm serve \
    furiosa-ai/Llama-3.1-8B-Instruct-FP8 \
    --tensor-parallel-size 4 \
    --pipeline-parallel-size 2 \
    --data-parallel-size 2 \
    --max-seq-len 1024 \
    --npu-queue-limit 1 \
    --estimation-time-limit-ms 5

python benchmarks/benchmark_serving.py \
    --backend vllm \
    --model $MODEL_ID \
    --port 8000 \
    --dataset-name sharegpt \
    --dataset-path $DATASET_PATH \
    --request-rate 16 \
    --num-prompts 10000 \
    --enable-device-monitor "npu" \
    --metric-percentiles "25,50,75,90,95,99" \
    --percentile-metrics "ttft,tpot,itl,e2el" \
    --save-result