ei]8 dZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z ddl Z ddl Z ddlmZmZmZddlmZmZmZddlmZddlmZ dd lmZn #e$rdZYnwxYweejZ e d z Z!ej"#d d Z$ej"#d Z%ej"#ddZ&e'ej"#ddZ(ej)ej*deZ+e+,edgddgdgGddeZ-GddeZ.de/de/fdZ0e+1ddZ2e+1d d!Z3e+#d"d#Z4e+#d$e%d&Z5e+6d'd(ed)e-fd*Z7e+6d+d(efd,Z8e9d-krtddl:Z:ddl;Z;ddlZ>d.Z?dPd0Z@dQd2ZAd3ZBe:jCd45ZDeDEd67ZFeFGd8d9:eFGd;d<:ZHeHId=d/d>?eFGd@dA:eFGdBdC:eFGdDdE:eFGdFdG:eDJZKeKjLd;kre@eKjMHdSeKjLd@kr eAdSeKjLdBkr ddl;Z;e;jNgdIdJKe!e dLz fD]6ZO eOPreOQ,#e$rY3wxYweRdMdS#e$rZSeRdNeSYdZS[SdSdZS[SwwxYweKjLdDkr ddl;Z;e;jNgdIdJKn #e$rYnwxYwe!e dLz fD]6ZO eOPreOQ,#e$rY3wxYwe@eTeKdOreKjMnd/HdSeKjLdFkr eBdSe?dSdS)RaC Simple LLM HTTP server using FastAPI. - Loads a local GGUF model once on startup (configurable via MODEL_PATH env var) - POST /ask accepts JSON { prompt, system_prompt?, generation_params? } and returns { response } - POST /shutdown optionally protected by LLM_SHUTDOWN_TOKEN env var (if set) - Writes .llm_server_pid in the project dir for compatibility with existing scripts Run: python3 llm_server.py Or run via: uvicorn llm_server:app --host 127.0.0.1 --port 5005 This server intentionally does NOT keep conversation state between requests: each /ask call is independent. N)Path)OptionalDictAny)FastAPI HTTPExceptionRequest) HTMLResponseStreamingResponseResponse)CORSMiddleware) BaseModel)Llamaz.llm_server_pid MODEL_PATHz\/home/asher/.lmstudio/models/lmstudio-community/gemma-3-1b-it-GGUF/gemma-3-1b-it-Q4_K_M.ggufLLM_SHUTDOWN_TOKENLLM_HOSTz0.0.0.0LLM_PORT5005z'%(asctime)s [%(levelname)s] %(message)s)levelformat*T) allow_originsallow_credentials allow_methods allow_headersc^eZdZUeed<dZeeed<dZeeee fed<dS) AskRequestpromptN system_promptgeneration_params) __name__ __module__ __qualname__str__annotations__rrr rr&/home/asher/github/pulse/llm_server.pyrr9sM KKK#'M8C='''26xS#X/66666r'rceZdZUeed<dS) AskResponseresponseN)r!r"r#r$r%r&r'r(r*r*>sMMMMMr'r*datareturnc|dS|d}g}|D]}|d|d|dzS)z Format a string as one or more SSE 'data:' lines, preserving blank lines. Ensures output ends with the required double-newline separator. Nzdata:  data:  )splitappendjoin)r,parts out_linesps r( format_sser8Asg  || JJt  EI ''!&&&& 99Y  & ((r'startupc ttjddStjdt }t|stjd|dS|ttjddttjddttjdd d d }tj d |d  tdi|tj _ tj dn=#t$r0}tjd|dtj _ Yd}~nd}~wwxYw t!jtj _n!#t$rdtj _YnwxYw tj}t(t-|tj d|dt(dS#t$rtjdYdSwxYw)Nz1llama_cpp not available; cannot start LLM server.rzModel path not found: LLM_N_CTX2048 LLM_N_THREADS4LLM_N_GPU_LAYERS0F) model_pathn_ctx n_threads n_gpu_layersverbosezLoading model from z ...zModel loaded successfully.zFailed to load model: %sz Wrote PID z to zFailed to write PID filer&)rloggingerrorosenvironget DEFAULT_MODELrexistsintinfoappstatellm Exception exceptionasyncioLockllm_lockgetpidPID_FILE write_textr$)rA llama_paramsepids r( startup_eventr]Ps0 } IJJJ m<>BJNN+=sCCDD L L7z777888-- --  12222 4a888  "$\^^  """! " 6ikkCHH%%% 5#5585566666 66645555556s=/E F &FF  "F00G GAH--IIshutdownc tr/ttjddSdS#t $rtjdYdSwxYw)NzRemoved PID file.z#Error removing PID file on shutdown)rXrLunlinkrFrNrRrSr&r'r(shutdown_eventra|sA ??   . OO    L, - - - - - . . AAA?@@@@@@AsAA A.-A.z/healthcXdtttjdddS)NokrQ)status model_loaded)boolgetattrrOrPr&r'r(healthrhs&DE41P1P,Q,Q R RRr'/)response_classc&d}t|S)Nu LLM Server

LLM Server






Response



        

        
News Q&A

        
Ask a concise question about recent news items — response streams below.

        

            

            
        

        
Answer

        


        
     
     
     )content)r
)htmls r(indexrnsj	DV%%%%r'z/askrequestreqc

Kttjdd

stdd|jdp|jr|jjnddd	}tjd	||jpd
}d|dd
|j
dgttjddt#tjddttjddttjddt#tjdddgd|jr|j|jddpd}d|vp|jddv}|r 
fd}t+|d S	ttjd!d}|
jd(d"i}n;|4d{V
jd(d"i}dddd{Vn#1d{VswxYwY|d#dd$d%}	|	r|		nd}
d&|
iS#t.$r8}tjd'|tdt3|d}~wwxYw))NrQizLLM not loadedstatus_codedetailzX-Forwarded-Forunknown,rzReceived /ask from %sz2You are a helpful assistant. Keep answers concise.system)rolerluserLLM_TEMPERATUREz0.2	LLM_TOP_K40	LLM_TOP_Pz0.95LLM_REPEAT_PENALTYz1.1LLM_MAX_TOKENS512z
<|eot_id|>)temperaturetop_ktop_prepeat_penalty
max_tokensstopacceptztext/event-streamstream)1truecKttjdd}|tj}|4d{V	jddd
D]}	|dpigd}|dp|dpi}d}t|tr|d	pd}nt|}|rYtjd
d|tj}|
r#d|d
WVtjdd{V#t$rYwxYwdWVn#t$r	jddi
}|r+d|vr'|dddd	nd}|pd
}d}t!dt#||D]0}d||||zd
WVtjdd{V1dWVnB#t$r5}	t%jd|	dt|	d
WVYd}	~	nd}	~	wwxYwYnwxYwdddd{VdS#1d{VswxYwYdS)NrVT)messagesrchoicesrdeltamessagerrlz^\s*assistant[:\s]*)flagsr0r1zevent: done
data: [DONE]

rz*LLM generation error (stream fallback): %szevent: error
data: r&)rgrOrPrTrUcreate_chat_completionrJ
isinstancedictr$resubIstripsleeprRrangelenrFrS)
lockchunkchoicer	text_partr+rl
chunk_sizeir[genrQrs
          r(event_streamzask..event_stream[s39j$77D||~~(
B(
B(
B(
B(
B(
B(
B(
B'B!;!;!bXVZ!b!b^a!b!b%%%&+ii	&:&:&BrdA%FF%+JJw$7$7$V6::i;P;P$VTVE(*I)%667,1IIi,@,@,FB		,/JJ	(;,.F3I2y`b`d,e,e,e	#,??#4#4!;*B9*B*B*B$B$B$B$B*1-*:*:$:$:$:$:$:$:$:(%%%$H%:99999 BBBB#=3#=#W#Wx#WSV#W#WV^#Aclpxcxcx(9"5a"8"C"G"G	"R"R"RA#*=b"7"7"9"9%(
!&q#g,,
!C!C33A"H71Qz\>+B"H"H"HHHHH")-"2"222222222======$BBB)*VXYZZZASVVAAAAAAAAAAAABB5(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
B(
BsIEC#D54E5
E?EEE
I
IB*HI
I+H;6I;IIIII
I&)I&)
media_typerVrrrrlr+zLLM generation error: %sr&)rgrOrPrheadersrJclienthostr2rrFrNrrfloatrHrIrMr updatequery_paramsrrrRrSr$)rorp	client_iprrstream_requestedrrr+rltextr[rrQrs            @@@r(askr:s
#)UD
)
)CF4DEEEE$$%677qSZSa>KI>AK
L
3LL
z	/shutdownctrP|jdp|jd}|tkrt	ddd}tjd|dd	iS)
NtokenzX-Shutdown-TokenizInvalid shutdown tokenrrctjd	trtn#t
$rYnwxYwt
jddS)Nz&Shutting down LLM server as requested.r)rFrNrXrLr`rRrH_exitr&r'r(rzshutdown.._exitsl=>>>	  
"!!!			D	
s2A		
AAg?rd
shutting_down)SHUTDOWN_TOKENrrJrr	threadingTimerstart)rorrs   r(r^r^sR$((11\W_5H5HI[5\5\N""C8PQQQQOC%%'''o&&r'__main__cTddl}|dttddS)Nrzllm_server:apprN)rport	log_level)uvicornrunHOSTPORT)rs r(start_foregroundrs,$4dfMMMMMr'/tmp/llm_server.logc	&trr	tt}tj|dtd|dS#t$rYnwxYwtj
ttdg}t|d5}tj|||tt }dddn#1swxYwYt#jd|Ztd|jd|	tt|jdS#t$rYdSwxYwtd	|dS)
Nrz Server already running with PID rab)stdoutstderrcwdg?zStarted background server (PID z), logging to z"Failed to start server. Check log:)rXrLrM	read_textrrHkillprintrRsys
executabler$__file__open
subprocessPopenAPP_DIRtimerpollr\rY)logfileexistingcmdoutr7s     r(start_backgroundrs??	
x113399;;<<!$$$CCCDDD



~s8}}g6
'4
 
 	PC S#g,,OOOA	P	P	P	P	P	P	P	P	P	P	P	P	P	P	P	
36688RAERRRRSSS
##CJJ/////




6@@@@@s6AA<<
B	B	=+C44C8;C8,E00
E>=E>crtstddS	tt}n;#t$r.tdtdYdSwxYw	tj	|tjt|dzD]=}tjd	tj	|d-#t$rYn#wxYwtj	|tjtd|d	n9#t $rtd
Ynt"$rtdYnwxYwtddS#tdwxYw)Nz-No PID file found; server may not be running.zInvalid PID file. Removing it.T)
missing_ok
g?rzStopped server (PID z).z+Process not found; removing stale PID file.z.Permission denied when trying to stop process.)rXrLrrMrrrRr`rHrsignalSIGTERMrrrSIGKILLProcessLookupErrorPermissionError)timeoutr\_s   r(stop_serverrs  	ABBBF	h((**002233CC			2333OOtO,,,FF		-GC(((7R<((
-
-
3GCOOOO EEV^,,,00001111!	A	A	A?@@@@@	D	D	DBCCCCC	D
OOtO,,,,,HOOtO,,,,sl8A%%4BB!AE)C?>E?
D
	ED

5EFE9FE96F8E99FF6c\tstddS	tt}t
j|dtd|ddS#t$rtdYdSwxYw)Nz&No PID file found; server not running.rz&Server appears to be running with PID .z?PID file exists but process not running. PID file may be stale.)	rXrLrrMrrrHrrR)r\s r(
status_serverrs  	:;;;F	Uh((**002233CGCOOOA3AAABBBBB	U	U	USTTTTTT	UsA BB+*B+zLLM server control)descriptionr)destrzStart server in foreground)helpzstart-bgz%Start server in background (detached)z--logzLog file for background server)defaultrrzStop server using PID filezstop-allz(Stop all running llm_server.py processeszrestart-allz5Stop all instances then start one background instancerdzShow server status)r)pkillz-fz
llm_server.pyF)checkz.llm_pidz/Requested stop for all llm_server.py processes.zFailed to stop all processes:log)r)r)U__doc__rmrHjsonrFrpathlibrtypingrrrrTrfastapirrr	fastapi.responsesr
rrfastapi.middleware.corsr
pydanticr	llama_cpprrRrresolveparentrrXrIrJrKrrrMrbasicConfigINFOrOadd_middlewarerr*r$r8on_eventr]rarhrnpostrr^r!argparserrrrrrrrArgumentParserparseradd_subparsersr
add_parsersbgadd_argument
parse_argsargsrrrr7rLr`rr[hasattrr&r'r(r	s				&&&&&&&&&&				3333333333GGGGGGGGGG222222EEE$x..
 
 
"
"
)&&
|.LMM
 455
z~~j),,
s2:>>*f--..',/XYYYY
gii%%%77777777
)
)S
)S
)
)
)
)i)6)6)6VjAAASSS\**m&m&+*m&^&^>>

..*Q.
R
RCW&;BbcccNN6