; $Id: dcs_sanitycheck.pro,v 1.17 2005/02/14 17:50:14 azuah Exp $
; Written by J.R.D. Copley.
;************************************************************************************************
;###############################################################################
;
; LICENSE:
;  The software in this file is written by an employee of 
;  National Institute of Standards and Technology 
;  as part of the DAVE software project.
;
;  The DAVE software package is not subject to copyright protection
;  and is in the public domain. It should be considered as an
;  experimental neutron scattering data reduction, visualization, and
;  analysis system. As such, the authors assume no responsibility
;  whatsoever for its use, and make no guarantees, expressed or
;  implied, about its quality, reliability, or any other
;  characteristic. The use of certain trade names or commercial
;  products does not imply any endorsement of a particular product,
;  nor does it imply that the named product is necessarily the best
;  product for the stated purpose. We would appreciate acknowledgment
;  if the DAVE software is used or if the code in this file is
;  included in another product.
;
;###############################################################################
pro dcs_sanitycheck,dcs_base,files_selected,nruns,nhedet,allchecks=allchecks,silent=silent,$
	bitflips,maxbitflips,$
	davePtr=davePtr,runtemperatures,message_str,$
	ratiocriterion=ratiocriterion,$
	skipdefaultchoices=skipdefaultchoices,$
	skiptchanmismatchmes=skiptchanmismatchmes,$
	logentry=logentry,$
	n_insane=n_insane,$
	dochecks=dochecks
;************************************************************************************************
; This routine loops over all runs, reads in to "davePtr", and performs
; various sanity checks.
;
; This is what the routine does, depending on the values of allchecks and silent.
; If allchecks=1 all sanity checks are performed.
; If allchecks=0 it simply looks for bit flips and stores the information in the array bitflips.
; If silent=1 there are no dialog messages and no prompting from the user.
;
compile_opt strictarr
;
;	The k'th sanity check is omitted if docheck[k]=0. There is no 0'th check, and check 1
; is mandatory; docheck[0] and docheck[1] are not actually used.
if (keyword_set(dochecks)) then docheck=[999,dochecks] else docheck=[999,1,1,1,0,1,1,1]
;
printinfo=0; used by dcs_readdatafile
filetype="binary"; used by dcs_readdatafile
;
; N.B. timsum is sum over detectors as function of time channel (size 1024)
;      detsum is sum over time as function of detector number (size 931)
;
; From an email to Nick, expressed in Octave language,
;
; grandsum(0)=sum(sum(histodata))
; grandsum(1)=sum(detsum(0:912))
; grandsum(2)=sum(sum(histohigh))
; grandsum(3)=sum(detsum(913:929))
; grandsum(4)=grandsum(0)+grandsum(2)
; grandsum(5)=sum(detsum)
; grandsum(6)=sum(timsum)
; grandsum(7)=sum(sum(totals))
;
;
fileread=strarr(nruns)
;
insane=intarr(nruns,7)
;
printlines=""
;
newbitflipmethod=0
;
for irun=1,nruns do begin
	irm1=irun-1
	messagelines=["Reading file"+strcompress(irun)+$
		" of"+strcompress(nruns),files_selected[irm1],"Please be patient..."]
	dcs_message_display,messagelines,base,group_leader=dcs_base,alignment=0
	omithisto=2
	dcs_readdatafile,files_selected[irm1],omithisto,printinfo,filetype,davePtr=davePtr
	;
	temp_setpoint=(*(*(*davePtr).dataStrPtr).specificPtr).temp_setpoint
	runtemperatures[irm1]=temp_setpoint > 0
	;
	fileread=files_selected[irm1]
	;
	; Sanity check # 1; look for bitflips.
	grandsum=(*(*(*davePtr).dataStrPtr).specificPtr).grandsum
	fail1=(grandsum[4] ne grandsum[5])
	fail2=(grandsum[4] ne grandsum[6])
	fail3=(grandsum[4] ne grandsum[7])
	;
	if (not (fail1 or fail2 or fail3)) then begin
		widget_control,base,/destroy
	endif else begin
		insane[irm1,0]=1
		lines=["File "+fileread,"  failed SANITY CHECK 1:",$
			"  grandsum[4] = "+strcompress(grandsum[4])]
		for i=1,3 do begin
			diff=grandsum[4]-grandsum[i+4]
			if (diff gt 0) then power=strcompress(alog(diff)/alog(2),/remove_all)
			if (diff lt 0) then power=strcompress("-"+alog(-diff)/alog(2),/remove_all)
			if (diff ne 0) then begin
				newline1="  grandsum["+strcompress(i+4,/remove_all)+$
					"] = "+strcompress(grandsum[i+4])
				newline2="  grandsum[4]-grandsum["+strcompress(i+4,/remove_all)+$
					"] = 2^"+strcompress(power)+" = "+strcompress(diff)+", i.e. "+$
					strcompress(dcs_dec2bin(diff))
				lines=[lines,newline1,newline2]
			endif
		endfor
;
		omithisto=0
		dcs_readdatafile,files_selected[irm1],omithisto,printinfo,filetype,davePtr=davePtr
		widget_control,base,/destroy
;
		if (newbitflipmethod) then begin
;
			result=dcs_bitflip_correct($
				(*(*(*davePtr).dataStrPtr).commonStr.histPtr).qty,$
				(*(*(*davePtr).dataStrPtr).specificPtr).histohigh,$
				(*(*(*davePtr).dataStrPtr).specificPtr).detsum,$
				(*(*(*davePtr).dataStrPtr).specificPtr).timsum,$
				bitflips=bitflip_irm1,maxbf=maxbitflips,inputfilename="",/onlydetect,printlevel=1)
			bitflips[*,*,irm1]=bitflip_irm1
;
		endif else begin
;
;	Sum He3 data and high end data over detectors and time channels
			tdh1=[long(total((*(*(*davePtr).dataStrPtr).commonStr.histPtr).qty,1,/double)),$
						long(total((*(*(*davePtr).dataStrPtr).specificPtr).histohigh,1,/double))]-$
						(*(*(*davePtr).dataStrPtr).specificPtr).detsum
			tdh2=long(total((*(*(*davePtr).dataStrPtr).commonStr.histPtr).qty,2,/double))+$
						long(total((*(*(*davePtr).dataStrPtr).specificPtr).histohigh,2,/double))-$
						(*(*(*davePtr).dataStrPtr).specificPtr).timsum
		;
			bad_t=where(tdh2 ne 0,tcount); disagreement in time sums over detector
			bad_d=where(tdh1 ne 0,dcount); disagreement in detector sums over time
;
			if (dcount eq 1 and tcount eq 1) then begin
				badt=bad_t[0]
				badd=bad_d[0]
				newline="  "+strcompress("Single bitflip occurred at [time,detector] = ["+$
					string(badt)+","+string(badd)+"].")
				lines=[lines,newline]
				bitflips[0,0,irm1]=badt
				bitflips[1,0,irm1]=badd
				bitflips[2,0,irm1]=tdh1[badd]
			endif else begin
				tdcount=0
				for it=0,tcount-1 do begin
					for id=0,dcount-1 do begin
						if (tdh2[bad_t[it]] eq tdh1[bad_d[id]]) then tdcount=tdcount+1
					endfor
				endfor
				if (tdcount le maxbitflips) then begin
					ibitflip=-1
					for it=0,tcount-1 do begin
						for id=0,dcount-1 do begin
							if (tdh2[bad_t[it]] eq tdh1[bad_d[id]]) then begin
								value=tdh1[bad_d[id]]
								newline=strcompress("  Bitflip occurred at [time,detector] = ["+$
									string(bad_t[it])+","+string(bad_d[id])+"]: value = "+string(value)+".")
								lines=[lines,newline]
								ibitflip=ibitflip+1
								bitflips[0,ibitflip,irm1]=bad_t[it]
								bitflips[1,ibitflip,irm1]=bad_d[id]
								bitflips[2,ibitflip,irm1]=tdh1[bad_d[id]]
							endif
						endfor
					endfor
				endif else begin
					lines=[lines,"  Time channels (0-1023) with disagreement in sums over detector number:",$
						"  "+strjoin(strcompress(bad_t[0:(20<(n_elements(bad_t)-1))])," :"),$
						"  Detector numbers (0-912) with disagreement in sums over time:",$
						"  "+strjoin(strcompress(bad_d[0:(20<(n_elements(bad_d)-1))])," :")]
				endelse
			endelse
;
		endelse
;
		if (allchecks and not silent) then begin
			blabel=['OK']
			def_button=0
			dcs_multichoice_modal,lines,decision,blabel,def_button,$
				group_leader=dcs_base,alignment=0
		endif
		printlines=[printlines,lines]
;
	endelse
;
	ch_ms=(*(*(*davePtr).dataStrPtr).specificPtr).ch_ms
	ch_srdenom=(*(*(*davePtr).dataStrPtr).specificPtr).ch_srdenom
	speriod=(60000000.0/ch_ms)*ch_srdenom
;
; Define the number of good time channels for the He3 detectors,
; ngtchan0, putting it in davePtr.
	twid=(*(*(*davePtr).dataStrPtr).specificPtr).tchanlook[0,*]*0.05
	times=[0,reform(total(twid,/cumulative))]
	tstart=times[0:(*(*(*davePtr).dataStrPtr).specificPtr).ntchan-1]
	tstop=times[1:(*(*(*davePtr).dataStrPtr).specificPtr).ntchan]
	chan=where(tstart le speriod-0.05 and tstop gt speriod-0.05) + 1
	ngtchan0=chan[0]
	(*(*(*davePtr).dataStrPtr).specificPtr).ngtchan0=ngtchan0
;
; Define the number of good time channels for the high end inputs,
; ngtchan1, putting it in davePtr.
	twid=(*(*(*davePtr).dataStrPtr).specificPtr).tchanlook[1,*]*0.05
	times=[0,reform(total(twid,/cumulative))]
	tstart=times[0:(*(*(*davePtr).dataStrPtr).specificPtr).ntchan-1]
	tstop=times[1:(*(*(*davePtr).dataStrPtr).specificPtr).ntchan]
	chan=where(tstart le speriod-0.05 and tstop gt speriod-0.05) + 1
	ngtchan1=chan[0]
	(*(*(*davePtr).dataStrPtr).specificPtr).ngtchan1=ngtchan1
;
;
	if (allchecks and docheck[2]) then begin
; Sanity check # 2; look for low and/or high end time channel mismatches..
		if (irun eq 1 and not skiptchanmismatchmes) then begin
			failed0=abs(speriod-tstart[ngtchan0]) gt 0.05
			failed1=abs(speriod-tstart[ngtchan1]) gt 0.05
			if (failed0 or failed1) then $
				lines=["File "+fileread,"  failed SANITY CHECK 2:"]
			if (failed0) then begin
				insane[irm1,1]=1
				lines=[lines,$
					"  The time between pulses at the sample,"+strcompress(speriod)+" us,",$
					"   does not correspond to a whole number of LOW END time channels.",$
					"  Time channel"+strcompress(chan[0]-1)+$
					" starts at"+strcompress(tstart[chan[0]-1])+" us"+$
					" and ends at"+strcompress(tstop[chan[0]-1])+" us.",$
					"  The data reduction may be compromised."]
				if (allchecks and not silent) then begin
					blabel=['OK']
					def_button=0
					dcs_multichoice_modal,lines,decision,blabel,def_button,$
						group_leader=dcs_base,alignment=0
				endif
			endif
;
			if (failed1) then begin
				insane[irm1,1]=1
				lines=[lines,$
					"  The time between pulses at the sample,"+strcompress(speriod)+" us,",$
					"   does not correspond to a whole number of HIGH END time channels.",$
					"  Time channel"+strcompress(chan[0]-1)+$
					" starts at"+strcompress(tstart[chan[0]-1])+" us"+$
					" and ends at"+strcompress(tstop[chan[0]-1])+" us.",$
					"  The data reduction may be compromised."]
				if (allchecks and not silent) then begin
					blabel=['OK']
					def_button=0
					dcs_multichoice_modal,lines,decision,blabel,def_button,$
						group_leader=dcs_base,alignment=0
				endif
			endif
			if (failed0 or failed1) then printlines=[printlines,lines]
		endif
	endif
;
	if (allchecks and docheck[3]) then begin
; Sanity check # 3. Look for counts in channels between firstbadtchan and lastbadtchan.
; First define firstbadtchan and lastbadtchan.
		firstbadtchan=(*(*(*davePtr).dataStrPtr).specificPtr).ngtchan0
		lastbadtchan=(*(*(*davePtr).dataStrPtr).specificPtr).ntchan-1
		if (firstbadtchan le lastbadtchan) then begin
			badtimes=where((*(*(*davePtr).dataStrPtr).specificPtr).timsum[firstbadtchan:lastbadtchan] gt 0, count)
			if (count gt 0) then begin
				insane[irm1,2]=1
				lines=["File "+fileread,"  failed SANITY CHECK 3:"]
				newline1="  Non-empty time channels (that should be empty)"+$
					strjoin(strcompress(firstbadtchan+badtimes))
				newline2="  Contents of these time channels:"+$
					strjoin(strcompress((*(*(*davePtr).dataStrPtr).specificPtr).timsum[firstbadtchan+badtimes]))
				lines=[lines,newline1,newline2]
				if (allchecks and not silent) then begin
					blabel=['OK']
					def_button=0
					dcs_multichoice_modal,lines,decision,blabel,def_button,$
						group_leader=dcs_base,alignment=0
				endif
				printlines=[printlines,lines]
			endif
		endif
	endif
;
; Sanity check # 4. Check that ratio of "fast chopper" counts to "slow chopper"
;		counts is essentially equal to frame overlap denominator.
	if (allchecks and docheck[4]) then begin
		denominator=(*(*(*davePtr).dataStrPtr).specificPtr).ch_input[4]
		and_counts=0
		ch_counts=lonarr(7)
		startchoice=strmid((*(*(*davePtr).dataStrPtr).specificPtr).startchoice,0,3)
		for k=0,16 do begin
			string123=strmid((*(*(*davePtr).dataStrPtr).specificPtr).highsource[k],0,3)
			if (string123 eq startchoice) then summed_counts=(*(*(*davePtr).dataStrPtr).specificPtr).nframes else $
				summed_counts=(*(*(*davePtr).dataStrPtr).specificPtr).detsum[913+k]
			if (string123 eq "AND") then and_counts=summed_counts
			if (string123 eq "CH1") then ch_counts[0]=summed_counts
			if (string123 eq "CH2") then ch_counts[1]=summed_counts
			if (string123 eq "CH3") then ch_counts[2]=summed_counts
			if (string123 eq "CH4") then ch_counts[3]=summed_counts
			if (string123 eq "CH5") then ch_counts[4]=summed_counts
			if (string123 eq "CH6") then ch_counts[5]=summed_counts
			if (string123 eq "CH7") then ch_counts[6]=summed_counts
		endfor
;
		quantity=abs(denominator*and_counts/ch_counts[6]-1.0)
		if (quantity gt ratiocriterion) then begin
			insane[irm1,3]=1
			lines=["File "+fileread,"  failed SANITY CHECK 4:"]
			newline1="  Frame overlap denominator = "+strcompress(denominator)
			newline2="  Chopper 7 counts = "+strcompress(ch_counts[6])
			newline3="  Coincidence counts = "+strcompress(and_counts)
			lines=[lines,newline1,newline2,newline3]
			if (allchecks and not silent) then begin
				blabel=['OK']
				def_button=0
				dcs_multichoice_modal,lines,decision,blabel,def_button,$
					group_leader=dcs_base,alignment=0
			endif
			printlines=[printlines,lines]
		endif
	endif
;
; Sanity check # 5. Look for invalid boards.
	if (allchecks and docheck[5]) then begin
		if ((*(*(*davePtr).dataStrPtr).specificPtr).badbrd ne 0) then begin
			insane[irm1,4]=1
			lines=["File "+fileread,"  failed SANITY CHECK 5:"]
			lines=[lines,"  INVALID BOARD"+strcompress((*(*(*davePtr).dataStrPtr).specificPtr).badbrd),$
				"","File "+fileread]
			if (allchecks and not silent) then begin
				blabel=['OK']
				def_button=0
				dcs_multichoice_modal,lines,decision,blabel,def_button,$
					group_leader=dcs_base,alignment=0
			endif
			printlines=[printlines,lines]
		endif
	endif
;
; Sanity check # 6. Look for invalid detectors.
	if (allchecks and docheck[6]) then begin
		if ((*(*(*davePtr).dataStrPtr).specificPtr).baddet ne 0) then begin
			insane[irm1,5]=1
			lines=["File "+fileread,"  failed SANITY CHECK 6:"]
			lines=[lines,"  INVALID DETECTOR"+strcompress((*(*(*davePtr).dataStrPtr).specificPtr).baddet),$
				"","File "+fileread]
			if (allchecks and not silent) then begin
				blabel=['OK']
				def_button=0
				dcs_multichoice_modal,lines,decision,blabel,def_button,$
					group_leader=dcs_base,alignment=0
			endif
			printlines=[printlines,lines]
		endif
	endif
;
; Sanity check # 7. Look for repeats and resets.
	if (allchecks and docheck[7]) then begin
		badboards_p0=where((*(*(*davePtr).dataStrPtr).specificPtr).repeats[0,*] gt 0, count_p0)
		badboards_r0=where((*(*(*davePtr).dataStrPtr).specificPtr).resets[0,*] gt 0, count_r0)
		badboards_p1=where((*(*(*davePtr).dataStrPtr).specificPtr).repeats[1,*] gt 0, count_p1)
		badboards_r1=where((*(*(*davePtr).dataStrPtr).specificPtr).resets[1,*] gt 0, count_r1)
;
		if (count_p0+count_r0+count_p1+count_r1 gt 0) then begin
			insane[irm1,6]=1
			lines=["File "+fileread,"  failed SANITY CHECK 7:"]
			line=3
			if (count_p0 gt 0) then begin
				lines=[lines,"  Nonzero repeats for crate 0 boards "+$
					strjoin(strcompress(long(badboards_p0)))]
				numbers=long((*(*(*davePtr).dataStrPtr).specificPtr).repeats[0,badboards_p0])
				if (count_p0 gt 1) then numbers=transpose(numbers)
				lines=[lines,"  Numbers of repeats are "+strjoin(strcompress(numbers))]
			endif
			if (count_r0 gt 0) then begin
				lines=[lines,"  Nonzero resets for crate 0 boards "+$
					strjoin(strcompress(long(badboards_r0)))]
				numbers=long((*(*(*davePtr).dataStrPtr).specificPtr).resets[0,badboards_r0])
				if (count_r0 gt 1) then numbers=transpose(numbers)
				lines=[lines,"  Numbers of resets are "+strjoin(strcompress(numbers))]
			endif
			if (count_p1 gt 0) then begin
				lines=[lines,"  Nonzero repeats for crate 1 boards "+$
					strjoin(strcompress(long(badboards_p1)))]
				numbers=long((*(*(*davePtr).dataStrPtr).specificPtr).repeats[1,badboards_p1])
				if (count_p1 gt 1) then numbers=transpose(numbers)
				lines=[lines,"  Numbers of repeats are "+strjoin(strcompress(numbers))]
			endif
			if (count_r1 gt 0) then begin
				lines=[lines,"  Nonzero resets for crate 1 boards "+strjoin(strcompress(long(badboards_r1)))]
				numbers=long((*(*(*davePtr).dataStrPtr).specificPtr).resets[1,badboards_r1])
				if (count_r1 gt 1) then numbers=transpose(numbers)
				lines=[lines,"  Numbers of resets are "+strjoin(strcompress(numbers))]
			endif
			if (allchecks and not silent) then begin
				blabel=['OK']
				def_button=0
				dcs_multichoice_modal,lines,decision,blabel,def_button,$
					group_leader=dcs_base,alignment=0
			endif
			printlines=[printlines,lines]
		endif
	endif
endfor
;
n_insane=total(insane,1)
if (total(n_insane) eq 0) then return
;
;If any of the files are "insane", decide whether or not to continue..
if (allchecks and silent or (not allchecks and not silent)) then decision=0 else $
	dcs_multiplechoice,dcs_base,decision,"Sanitycheckprint",message_str,60,$
		skipdefaultchoices=skipdefaultchoices
;
nlines=n_elements(printlines)
if (nlines gt 0 and decision eq 0) then begin
	for i=0,nlines-1 do begin
		logentry=logentry+string(10B)+printlines[i]
	endfor
endif
openw,unit,!logfile,/get_lun
printf,unit,logentry
free_lun,unit
;
if (nlines gt 0 and decision eq 1) then begin
	tempfile="temp_print.txt"
	openw,unit,tempfile,/get_lun
	for i=0,nlines-1 do begin
		printf,unit,printlines[i]
	endfor
	free_lun,unit
	dcs_print_textfile,file=tempfile
	file_delete,tempfile
endif
;
if (nlines gt 0 and decision eq 2) then begin
	for i=0,nlines-1 do begin
		if (!debug) then print,printlines[i]
	endfor
endif
;
if (allchecks and not silent) then begin
	dcs_multiplechoice,dcs_base,decision,"Sanitycontinue",message_str,60,$
		skipdefaultchoices=skipdefaultchoices
	if (decision eq 1) then nruns=0
endif
;
end
