o Zh3@sddlmZddlZddlmZmZmZmZmZm Z m Z m Z m Z m Z ddlZddlZddlmZmZddlmZddlmZerNddlmZmZddlmZGd d d eZdS) ) annotationsN) TYPE_CHECKINGAny AsyncIteratorDict GeneratorIteratorListMappingOptionalUnion)AsyncCallbackManagerForLLMRunCallbackManagerForLLMRun)LLM)GenerationChunk)RESTfulChatModelHandleRESTfulGenerateModelHandle)LlamaCppGenerateConfigcseZdZUdZdZded<ded< ded< ded <   d9d:fd d Zed;ddZedd?ddZ  d>d@d%d&Z  d>dAd(d)Z dBdCd,d-ZedDd0d1Z  d>dEd4d5Z dBdFd7d8ZZS)G Xinferencea `Xinference` large-scale model inference service. To use, you should have the xinference library installed: .. code-block:: bash pip install "xinference[all]" If you're simply using the services provided by Xinference, you can utilize the xinference_client package: .. code-block:: bash pip install xinference_client Check out: https://github.com/xorbitsai/inference To run, you need to start a Xinference supervisor on one server and Xinference workers on the other servers Example: To start a local instance of Xinference, run .. code-block:: bash $ xinference You can also deploy Xinference in a distributed cluster. Here are the steps: Starting the supervisor: .. code-block:: bash $ xinference-supervisor Starting the worker: .. code-block:: bash $ xinference-worker Then, launch a model using command line interface (CLI). Example: .. code-block:: bash $ xinference launch -n orca -s 3 -q q4_0 It will return a model UID. Then, you can use Xinference with LangChain. Example: .. code-block:: python from langchain_community.llms import Xinference llm = Xinference( server_url="http://0.0.0.0:9997", model_uid = {model_uid} # replace model_uid with the model UID return from launching the model ) llm.invoke( prompt="Q: where can we visit in the capital of France? A:", generate_config={"max_tokens": 1024, "stream": True}, ) Example: .. code-block:: python from langchain_community.llms import Xinference from langchain.prompts import PromptTemplate llm = Xinference( server_url="http://0.0.0.0:9997", model_uid={model_uid}, # replace model_uid with the model UID return from launching the model stream=True ) prompt = PromptTemplate( input=['country'], template="Q: where can we visit in the capital of {country}? A:" ) chain = prompt | llm chain.stream(input={'country': 'France'}) To view all the supported builtin models, run: .. code-block:: bash $ xinference list --all Nz Optional[Any]client Optional[str] server_url model_uidzDict[str, Any] model_kwargsapi_keyrc szddlm}Wn#ty+zddlm}Wnty(}ztd|d}~wwYnw|p/i}tjd i|||d|jdurFtd|jdurOtdi|_ d|_ | |durh|j rhd||j d <||||_ dS) Nr) RESTfulClientzCould not import RESTfulClient from xinference. Please install it with `pip install xinference` or `pip install xinference_client`.rrrzPlease provide server URLzPlease provide the model UIDFzBearer Authorization) xinference.clientr ImportErrorZxinference_clientsuper__init__r ValueErrorr_headers_cluster_authed_check_cluster_authenticatedr)selfrrrrre __class__rZ/var/www/html/lang_env/lib/python3.10/site-packages/langchain_community/llms/xinference.pyr"s@    zXinference.__init__returnstrcCsdS)zReturn type of llm.Z xinferencerr'rrr+ _llm_typeszXinference._llm_typeMapping[str, Any]cCs"id|jid|jid|jiS)zGet the identifying parameters.rrrrr.rrr+_identifying_paramsszXinference._identifying_paramsNonecCsd|jd}t|}|jdkrd|_dS|jdkr%td|d|}t|d|_dS)Nz/v1/cluster/authFz+Failed to get cluster information, detail: detailauth)rrequestsget status_coder% RuntimeErrorjsonbool)r'urlresponseZ response_datarrr+r&s      z'Xinference._check_cluster_authenticatedpromptstopOptional[List[str]] run_manager"Optional[CallbackManagerForLLMRun]kwargsc Ks|jdur td|j|j}|di}i|j|}|r#||d<|r>|dr>d}|j||||dD]}||7}q5|S|j||d} | d d d S) aqCall the xinference model and return the output. Args: prompt: The prompt to use for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Returns: The generated string by the model. NClient is not initialized!generate_configr@stream)modelr?rBrFr?rFchoicesrtext)rr# get_modelrr8r_stream_generategenerate) r'r?r@rBrDrIrFZcombined_text_outputtoken completionrrr+_calls&   zXinference._callrI=Union['RESTfulGenerateModelHandle', 'RESTfulChatModelHandle']rF"Optional['LlamaCppGenerateConfig']Generator[str, None, None]c cs||j||d}|D]1}t|tr;|dg}|r;|d}t|tr;|dd} |d} |r8|j| |j| d| Vq dS) a^ Args: prompt: The prompt to use for generation. model: The model used for generation. stop: Optional list of stop words to use when generating. generate_config: Optional dictionary for the configuration used for generation. Yields: A string token. rJrKrrLrHlogprobs)rPverbose log_probsN)rO isinstancedictr8on_llm_new_tokenrW) r'rIr?rBrFZstreaming_responsechunkrKchoicerPrXrrr+rNs&     zXinference._stream_generateIterator[GenerationChunk]cksh|di}i|j|}|r||d<|||D]}|r1||}|r.|j|j|jd|VqdSNrFr@)rW)r8r_create_generate_stream$_stream_response_to_generation_chunkr[rLrWr'r?r@rBrDrFZ stream_respr\rrr+_streams   zXinference._streamOptional[Dict[str, List[str]]] Iterator[str]ccs:|jdur td|j|j}|j||dEdHdS)NrErJ)rr#rMrrO)r'r?rFrIrrr+r`.s  z"Xinference._create_generate_streamstream_responsercCszd}t|tr9|dg}|r4|d}t|tr0|dd}t|t|dd|dddd Std t|d Std ) z0Convert a stream response to a generation chunk.rHrKrrL finish_reasonNrV)rgrV)rLZgeneration_infozchoice type error!)rLzstream_response type error!)rYrZr8r TypeError)rfrPrKr]rrr+ra6s"       z/Xinference._stream_response_to_generation_chunk'Optional[AsyncCallbackManagerForLLMRun]AsyncIterator[GenerationChunk]cKsx|di}i|j|}|r||d<|||2z3dHW}|r8||}|r5|j|j|jdIdH|Vq6dSr_)r8r_acreate_generate_streamrar[rLrWrbrrr+_astreamQs    zXinference._astreamAsyncIterator[str]c Csh|j|d}|dur|D]\}}|||<qt|o|d}t4IdH~}|j|jd|d4IdHX}|jdkrV|jdkrHt d|j } t d|jd | |j 2z,3dHW} |sht | VqY| d } | d r| td d} | sqYt | VqY6WdIdHn 1IdHswYWdIdHdS1IdHswYdS) N)rIr?rGz/v1/completions)r=r;r4r3z)astream call failed with status code 404.z%astream call failed with status code z . Details: zutf-8sdata:)ritemsr<r8aiohttpZ ClientSessionpostrstatusFileNotFoundErrorrLr#contentr;loadsdecode startswithlenstrip) r'r?rF request_bodykeyvaluerGsessionr>Zoptional_detaillineZjson_strrrr+rkfsH         *.z#Xinference._acreate_generate_stream)NNN)rrrrrrrr)r,r-)r,r0)r,r2)NN) r?r-r@rArBrCrDrr,r-) rIrSr?r-rBrCrFrTr,rU) r?r-r@rArBrCrDrr,r^)N)r?r-rFrdr,re)rfr-r,r) r?r-r@rArBrirDrr,rj)r?r-rFrdr,rm)__name__ __module__ __qualname____doc__r__annotations__r"propertyr/r1r&rRrNrcr` staticmethodrarlrk __classcell__rrr)r+rsH  \*    0 &    r) __future__rr;typingrrrrrrr r r r ror7Zlangchain_core.callbacksr rZ#langchain_core.language_models.llmsrZlangchain_core.outputsrrrrZxinference.model.llm.corerrrrrr+s 0