; $Id: dave_parse_ice.pro,v 1.12 2007/05/08 17:27:55 azuah Exp $
; ************************************ ;
; NAME:
;  DAVE_PARSE_ICE
;
; PURPOSE:
;  This function parses and ICE data file and
;  returns the data contained therein in a structure
;  accessed via an output parameter DSTRUCT.  The input
;  parameter is DSTRING which is a string array containing
;  the entire contents of the data file.
;
; CATEGORY:
;  DAVE, Data Reduction, TAS.
;
; AUTHOR:
;   Robert M. Dimeo, Ph.D.
;   NIST Center for Neutron Research
;   100 Bureau Drive
;   Gaithersburg, MD 20899
;   Phone: (301) 975-8135
;   E-mail: robert.dimeo@nist.gov
;   http://www.ncnr.nist.gov/staff/dimeo
;
;   Major modifications by:
;   Richard Tumanjong Azuah
;
; LICENSE:
;  The software in this file is written by an employee of
;  National Institute of Standards and Technology
;  as part of the DAVE software project.
;
;  The DAVE software package is not subject to copyright protection
;  and is in the public domain. It should be considered as an
;  experimental neutron scattering data reduction, visualization, and
;  analysis system. As such, the authors assume no responsibility
;  whatsoever for its use, and make no guarantees, expressed or
;  implied, about its quality, reliability, or any other
;  characteristic. The use of certain trade names or commercial
;  products does not imply any endorsement of a particular product,
;  nor does it imply that the named product is necessarily the best
;  product for the stated purpose. We would appreciate acknowledgment
;  if the DAVE software is used of if the code in this file is
;  included in another product.
; ************************************ ;
function dave_parse_ice, dstring,dstruct    $
  ,header=header_string $
  ,indep_var_pos=indep_var_pos $
  ,dep_var_pos=dep_var_pos $
  ;;,plot_indep_var = plot_indep_var $
  ,lattice=lattice $
  ,fixedEType=fixedEType $
  ,fixedEValue=fixedEValue $
  ,dSpaceM=dSpaceM $
  ,dSpaceA=dSpaceA $
  ,orientation=orientation $
  ,sdesc=sdesc $
  ,anaPkgStr=anaPkgStr $
  ,errmsg = errmsg

errmsg = ''
catch,the_error
if the_error ne 0 then begin
   catch,/cancel
   errmsg = !error_state.msg
   return,0B
endif
nlines = n_elements(dstring)

header_counter = 0L
data_counter = 0L

; Go through each element of the string array and get out the first data line
; and the last data line
last_line_boolean = 0B
indep_index = -1
dep_index = -1
latt_index = -1
dsmon_index = -1
dsana_index = -1
fEi_index = -1
orient_index = -1
sdesc_index = -1
lattice = [0.0D,0.0D,0.0D,0.0D,0.0D,0.0D]
orientation = [0.0,0.0,0.0,0.0,0.0,0.0]
fixedEType = ''
fixedEValue = 0.0
dspaceM = 0.0
dspaceA = 0.0
sdesc = ''
anapkg = 0

for i = 0,nlines-1 do begin
    if (strmid(dstring[i],0,1) ne '#') then continue ; don't analyze data lines
    
    toks = strsplit(dstring[i],' ',/extract,count=ntoks)
    
    case strupcase(toks[0]) of
        '#COLUMNS': begin
            first_data_line = i+1
            titles = toks[1:ntoks-1]
            ncols = ntoks - 1
        end
        '#COMMENT':
        '#SCAN': begin
            indep_var_pos = fix(toks[1]) - 1
            indep_index = (indep_var_pos lt 0)? -1 : 1 
        end
        '#SIGNAL': begin
            dep_var_pos = fix(toks[1]) - 1 ; zero-based index
            dep_index = (dep_var_pos lt 0)? -1 : 1 
        end
        '#LATTICE': if (ntoks eq 7) then lattice =double([toks[1],toks[2],toks[3],toks[4],toks[5],toks[6]])
        '#FIXEDE': begin
            if (ntoks eq 3) then begin
                fixedEType = toks[1]
                fixedEValue = float(toks[2])
            endif
        end
        '#MONOSPACING': if (ntoks eq 2) then  dspaceM = float(toks[1])
        '#ANASPACING': if (ntoks eq 2) then  dspaceA = float(toks[1])
        '#ORIENT':   if (ntoks eq 7) then orientation = float([toks[1],toks[2],toks[3],toks[4],toks[5],toks[6]])
        '#SCANDESCR':   if (ntoks eq 2) then sdesc = toks[1]
        '#ANALYZERDETECTORMODE': begin
            if (ntoks eq 3) then begin
                anaPkg = 1
                anaMode = fix(toks[1])
                anaModeName = strtrim(toks[2],2)
            endif
        end
        '#ANALYZERDETECTORDEVICESOFINTEREST': anaSigDets = toks[1:ntoks-1]
        '#ANALYZERSDGROUP': anaSDDets = toks[1:ntoks-1]
        '#ANALYZERDDGROUP': anaDDDets = toks[1:ntoks-1]
        '#ANALYZERDOORDETECTORGROUP': anaTDDets = toks[1:ntoks-1]
        '#ANALYZERPSDGROUP': anaPSDDets = toks[1:ntoks-1]
        
        else: begin
            ;; check for fit parameters present in some scans at end
            ;; of file
            if (strmatch(toks[0],'*:END*',/fold_case)) then $
              last_data_line = i-1
        end
    endcase
    

;    if strpos(strupcase(this[0]),'#COLUMNS') ne -1 then first_data_line = i+1
;    if strpos(dstring[i],':END') ne -1 then begin
;       if ~last_line_boolean then last_data_line = i-1
;       last_line_boolean = 1B
;       ;last_data_line = i-1
;    endif
;    col_pos = strpos(strupcase(dstring[i]),'#COLUMNS')
;    ncol_pos = strpos(strupcase(dstring[i]),'#NCOLUMNS')
;    comment_pos = strpos(strupcase(dstring[i]),'#COMMENT')
;    indep_var_pos = strpos(strupcase(dstring[i]),'#SCAN ')
;    dep_var_pos = strpos(strupcase(dstring[i]),'#SIGNAL ')
;    latt_pos = strpos(strupcase(dstring[i]),'#LATTICE ')

;    fEi_pos = strpos(strupcase(dstring[i]),'#FIXEDE')
;    dsmon_pos = strpos(strupcase(dstring[i]),'#MONOSPACING')
;    dsana_pos = strpos(strupcase(dstring[i]),'#ANASPACING')
;    orient_pos = strpos(strupcase(dstring[i]),'#ORIENT')
;    sdesc_pos = strpos(strupcase(dstring[i]),'#SCANDESCR')
;    anamode_pos = strpos(strupcase(dstring[i]),'#ANALYZERDETECTORMODE')
;    anaDets_pos= strpos(strupcase(dstring[i]),'#ANALYZERDETECTORDEVICESOFINTEREST')
;    anaSD_pos = strpos(strupcase(dstring[i]),'#ANALYZERSDGROUP')
;    anaDD_pos = strpos(strupcase(dstring[i]),'#ANALYZERDDGROUP')
;    anaTD_pos = strpos(strupcase(dstring[i]),'#ANALYZERDOORDETECTORGROUP')
;    anaPSD_pos = strpos(strupcase(dstring[i]),'#ANALYZERPSDGROUP')

;    if indep_var_pos ne -1 then indep_index = i
;    if dep_var_pos ne -1 then dep_index = i
;    if comment_pos ne -1 then comment_index = i
;    if ncol_pos ne -1 then ncol_index = i
;    if col_pos ne -1 then title_index = i
;    if latt_pos ne -1 then latt_index = i

;    if fEi_pos ne -1 then fEi_index = i
;    if dsmon_pos ne -1 then dsmon_index = i
;    if dsana_pos ne -1 then dsana_index = i
;    if orient_pos ne -1 then orient_index = i
;    if sdesc_pos ne -1 then sdesc_index = i

endfor


;result = strsplit(dstring[ncol_index],/extract)
;ncols = fix(result[1])
;;;titles = strsplit(dstring[title_index],/extract)
;;;ncols = n_elements(titles) - 1
;;;titles = titles[1:ncols]

;;;lattice = [0.0D,0.0D,0.0D,0.0D,0.0D,0.0D]
;;;if (latt_index ne -1) then begin
;;;   toks = strsplit(dstring[latt_index],/extract,count=ntoks)
;;;   if (ntoks eq 7) then $
;;;      lattice =double([toks[1],toks[2],toks[3],toks[4],toks[5],toks[6]])
;;;endif
;;;orientation = [0.0,0.0,0.0,0.0,0.0,0.0]
;;;if (orient_index ne -1) then begin
;;;   toks = strsplit(dstring[orient_index],/extract,count=ntoks)
;;;   if (ntoks eq 7) then $
;;;      orientation =float([toks[1],toks[2],toks[3],toks[4],toks[5],toks[6]])
;;;endif
;;;fixedEType = ''
;;;fixedEValue = 0.0
;;;if (fEi_index ne -1) then begin
;;;   toks = strsplit(dstring[fEi_index],/extract,count=ntoks)
;;;   if (ntoks eq 3) then begin
;;;      fixedEType = toks[1]
;;;      fixedEValue = float(toks[2])
;;;   endif
;;;endif
;;;dspaceA = 0.0
;;;if (dsana_index ne -1) then begin
;;;   toks = strsplit(dstring[dsana_index],/extract,count=ntoks)
;;;   if (ntoks eq 2) then  dspaceA = float(toks[1])
;;;endif
;;;dspaceM = 0.0
;;;if (dsmon_index ne -1) then begin
;;;   toks = strsplit(dstring[dsmon_index],/extract,count=ntoks)
;;;   if (ntoks eq 2) then  dspaceM = float(toks[1])
;;;endif
;;;sdesc = ''
;;;if (sdesc_index ne -1) then begin
;;;   toks = strsplit(dstring[sdesc_index],/extract,count=ntoks)
;;;   if (ntoks eq 2) then sdesc = toks[1]
;;;endif
; change 'MONITOR' to 'MON' consistent with ICP format
;ind = where(strupcase(titles) eq 'MONITOR',cnt)
;if (cnt eq 1) then titles[ind] = 'Mon'

if n_elements(last_data_line) eq 0 then $
   last_data_line = nlines-1

data_string = dstring[first_data_line:last_data_line]
ndata = last_data_line-first_data_line+1
data = fltarr(ncols,ndata)
for i = 0,ndata-1 do begin
    tmpBuf = strsplit(data_string[i],' ',/extract)
    tmpInd = where(tmpBuf eq 'N/A',indCnt)
    if (indCnt gt 0) then tmpBuf[tmpInd] = 0.0 ;'-999'
    data[*,i] = float(tmpBuf)
endfor

header_indices = where((indgen(nlines) ge last_data_line) or $
   (indgen(nlines) le first_data_line),count_header)
header_string = dstring[header_indices]

; The indep variable
;;;if (indep_index ne -1) then begin
;;;   toks = strsplit(dstring[indep_index],/extract)
;;;   indep_var_pos = fix(toks[1]) - 1
;;;   if (indep_var_pos lt 0) then indep_index = -1 
;;;endif
if (indep_index eq -1) then begin
    ;; find first varying device
    i = -1
    found = -1
    small = 0.001
    while ((++i lt ncols) && (found eq -1)) do begin
        diff = abs(data[i,ndata-1] - data[i,0])
        colHeader = strupcase(titles[i])
        notAMon = colHeader ne 'MONITOR' 
        notADet = colHeader ne 'DETECTOR'
        notTime = colHeader ne 'TIME'
        found = ((diff gt small) && notAMon && notADet && notTime)? i : -1
    endwhile
    
    if (found gt -1) then begin
;        plot_indep_var = titles[found]
        indep_var_pos = found
    endif else begin
;        plot_indep_var = ''
        indep_var_pos = 0
    endelse
endif

; The dep variable. If the signal was not explicitly specified, use the detector
if (dep_index eq -1) then begin
    dep_var_pos = where(strupcase(titles) eq 'DETECTOR')
    dep_var_pos = dep_var_pos[0] - 1 ; zero-based index
endif ;;;else begin
;;;   toks = strsplit(dstring[dep_index],/extract)
;;;   dep_var_pos = fix(toks[1]) - 1 ; zero-based index
;;;endelse

;; Previously, the dependent column was moved to the last column of the
;; structure which is the default location for the Signal/Counts.
;; Now (May, 07) alter this behaviour by also adding an extra column
;; called 'Signal' and _copy_ the dependent data to it.
theRest = where(indgen(ncols) ne dep_var_pos)
data = [data[theRest,*],data[dep_var_pos,*]]
titles = [titles[theRest],titles[dep_var_pos]]
if (indep_var_pos gt dep_var_pos) then indep_var_pos-- ; shift by 1 if necessary after rearrangement
dep_var_pos = ncols - 1

data = [data,data[dep_var_pos,*]]
titles = [titles,'Signal']
nCols++

; Processing for analyzer package
if (anaPkg) then begin
    ;; sum data from package detectors in use
    ndets = n_elements(anaSigDets)
    for i = 0,ndets-1 do begin
        det_pos = where(titles eq anaSigDets[i],cnt)
        if (cnt) then $
          det_sum = (n_elements(det_sum) gt 0)? det_sum+reform(data[det_pos,*]) : reform(data[det_pos,*])
    endfor
    
    ;; By default, the signal column should be the sum of detectors in
    ;; use
    data[nCols-1,*] = det_sum
    
    ;; record analyzer info in a structure
    anaPkgStr = {mode:anaMode $
                 ,signalDets:anaSigDets $
                 ,sigDetsInSum:strjoin(anaSigDets,';',/single) $
                 ,TDDetsInSum:strjoin(anaTDDets,';',/single) $
                 ,SDDets:anaSDDets $
                 ,DDDets:anaDDDets $
                 ,TDDets:anaTDDets $
                 ,PSDDets:anaPSDDets $
                }
endif





; Add an error column
error = reform(data[nCols-1,*]) ;reform(data[dep_var_pos,*])
badInd = where(error le 0.0,cnt)
if (cnt gt 0) then error[badInd] = 1.0

; Create a structure for the data with the titles defined
; as the tag names.
dstruct = create_struct(titles[0],ptr_new(reform(data[0,*])))
for i = 1,nCols-1 do dstruct = create_struct(dstruct,titles[i],ptr_new(reform(data[i,*])))
dstruct = create_struct(dstruct,'ERROR',ptr_new(sqrt(error)))

; Add an INDEX column - expected in reduction section of code
dstruct = create_struct('INDEX',ptr_new(findgen(ndata)),dstruct)
indep_var_pos++                 ; accommodate INDEX
dep_var_pos++                   ; accommodate INDEX

catch,/cancel
return,1B
end
; ************************************ ;
pro test_read_ice
  filename = sourceroot()+'blah57.bt7' ;'PrCMO6024.bt7';'fpx6013.bt7' ; Qscan02820.btx
read_ok = dave_read_ice(filename,dstring = dstring,errmsg = errmsg)
if ~read_ok then begin
   print,errmsg
   return
endif
parse_ok = dave_parse_ice(dstring,dstruct,header=header $
                          ,indep_var_pos=indep_ind $
                          ,dep_var_pos=dep_ind $
                          ,lattice=lattice $
                          ,fixedEType=fixedEType $
                          ,fixedEValue=fixedEValue $
                          ,dSpaceM=dSpaceM $
                          ,dSpaceA=dSpaceA $
                          ,orientation=orientation $
                          ,sdesc=sdesc $
                          ,anaPkgStr=anaPkgStr $
                          ,errmsg = errmsg)
if ~parse_ok then print,errmsg

if ~parse_ok then return

names = tag_names(dstruct)
x = (*dstruct.(indep_ind))
y = (*dstruct.(dep_ind))
dy = (*dstruct.error)
plot,x,y,psym = 4
errplot,x,y-dy,y+dy,width = 0.0

heap_free, dstruct
end
