PRO dustem_check_data,data_sed,data_ext,sed,ext,polext,polsed,polfrac,psi_em,qsed,used,qext,uext,psi_ext,fpolext

;+
; NAME:

;    dustem_check_data
;

; PURPOSE:

;    Verify that the input data conforms to what DustEMWrap requires
;

; CATEGORY:

;    DustEMWrap, Mid-Level, Distributed, Initialization
;

; CALLING SEQUENCE:
;    dustem_check_data,data_sed,data_ext,sed,ext,polext,polsed,polfrac,psi_em,qsed,used,qext,uext,psi_ext,fpolext,[/help]

;

; INPUTS:
;    data_sed : Emission dustemwrap structure
;    data_ext : Extinction dustemwrap structure

;

; OPTIONAL INPUT PARAMETERS:
;    None

;

; OUTPUTS:
;    ## adapting the data structures to the format used by the fitting algorithm 
;    sed : emission formatted structure 
;    ext : extinction formatted structure
;    polext : polarized extinction formatted structure
;    polsed : polarized emission formatted structure 
;    polfrac : fractional polarized emission formatted structure
;    psi_em  : emission polarization angle formatted structure 
;    qsed : stokesQ emission formatted structure
;    used : stokesU emission formatted structure
;    qext : stokesQ extinction formatted structure
;    uext : stokesU extinction formatted structure
;    psi_ext : extinction polarization angle formatted structure
;    fpolext : fractional extinction polarization formatted structure

;

; OPTIONAL OUTPUT PARAMETERS:
;    None

;

; ACCEPTED KEY-WORDS:
;    help      = If set print this help

;

; COMMON BLOCKS:
;    None

;

; SIDE EFFECTS:

;

; RESTRICTIONS:

;    The DustEMWrap IDL code must be installed
;

; PROCEDURE:
;    None

;

; EXAMPLES
;    dustem_check_data,data_sed,data_ext,sed,ext,polext,polsed,polfrac,psi_em,qsed,used,qext,uext,psi_ext,fpolext,[/help]

;
; MODIFICATION HISTORY:
;    Written I Choubani 2022
;    Evolution details on the DustEMWrap gitlab.
;    See http://dustemwrap.irap.omp.eu/ for FAQ and help.   

;-


IF keyword_set(help) THEN BEGIN
  doc_library,'dustem_check_data'
  goto,the_end
ENDIF

;CHECKING THE FORMAT OF THE SED STRUCTURE

IF KEYWORD_SET(data_sed) THEN BEGIN
            

    ans=''

    wavs=data_sed.wave

    ;=== Impose central wavelengths for photometric channels

    ind=where(data_sed.filter NE 'SPECTRUM',count)

    IF count NE 0 THEN BEGIN

        wavs(ind)=dustem_filter2wav(data_sed(ind).filter)

    ENDIF

    ;=== Initializing observation structure

    obs_EM={instru_names:data_sed.instru,filt_names:data_sed.filter,wav:wavs, values:data_sed.StokesI,sigma:sqrt(abs(data_sed.sigmaII))} 

    ;INITIALIZING OUTPUT STRUCTURES

    sed = obs_EM

    qsed = sed
    used = sed

    ;Test if there is StokesI data. If not move to polsed testing.

    ind = where(data_sed.StokesI NE la_undef() and finite(data_sed.StokesI) EQ 1 and data_sed.sigmaII NE la_undef() and finite(data_sed.sigmaII) EQ 1,N_sed) ;Testing if the data and errors are set

    IF N_sed EQ 0 THEN BEGIN

        sed = ptr_new()

        goto, qzone

    ENDIF ELSE BEGIN 

        ind = where(finite(data_sed.StokesI) EQ 0 or finite(data_sed.sigmaII) EQ 0, countnoki) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
        IF countnoki NE 0 THEN BEGIN
            message, "SED format isn't met. NaN value(s) detected in StokesI data.",/continue
            message, 'Without automatic format fitering you cannot proceed.',/continue
            ;stop
            read, ans, prompt='Would you like automatic format filtering? (Y/N):'
            IF strupcase(ans) EQ 'N' THEN BEGIN
                message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
                message, 'Aborting ...'
            ENDIF
        ENDIF 
        IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
        ELSE message, 'Data format is met. Removal of undefined data points...',/continue
        tdaerri=where((data_sed.sigmaII EQ la_undef() and data_sed.StokesI NE la_undef()) or (data_sed.sigmaII NE la_undef() and data_sed.StokesI EQ la_undef()),ctdaerri)
        IF ctdaerri NE 0 THEN BEGIN
            message, 'Data and errors do not match. Undefinded data detected. Aborting...',/continue

        ENDIF

        testi=where(data_sed.sigmaII NE 0. and finite(data_sed.sigmaII) EQ 1 and data_sed.sigmaII NE la_undef() and data_sed.StokesI NE la_undef() and finite(data_sed.StokesI) EQ 1,ctesti)                
        IF ctesti NE 0 THEN BEGIN 
            instru_names = sed.instru_names(testi)
            filt_names = sed.filt_names(testi)
            wav = sed.wav(testi)
            values = sed.values(testi)
            sigma = sed.sigma(testi)
            sed={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
        ENDIF    

    ENDELSE

    qzone:
    ans=''

    qsed.values=data_sed.StokesQ
    qsed.sigma=sqrt(abs(data_sed.sigmaQQ))

    ;Testig if there is StokesQ data. If not move to used data

    ind = where(data_sed.StokesQ NE la_undef() and finite(data_sed.StokesQ) EQ 1 and data_sed.sigmaQQ NE la_undef() and finite(data_sed.sigmaQQ) EQ 1,NQ)

    IF NQ EQ 0 THEN BEGIN

        qsed = ptr_new()

        goto, uzone
    ENDIF ELSE BEGIN

    ind = where(finite(data_sed.StokesQ) EQ 0 or finite(data_sed.sigmaQQ) EQ 0, countnokq) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 

    IF countnokq NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in StokesQ data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue

            message, 'Aborting ...'

        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue

    tdaerrq=where((data_sed.sigmaQQ EQ la_undef() and data_sed.StokesQ NE la_undef()) or (data_sed.sigmaQQ NE la_undef() and data_sed.StokesQ EQ la_undef()),ctdaerrq)

    IF ctdaerrq NE 0 THEN BEGIN
        message, 'Data and errors in Stokes Q do not match. Undefinded data detected for these filters:',/continue
        print,data_sed[tdaerrq].filter
        ;message, 'Data and errors do not match. Undefinded data detected. Aborting...',/continue
        stop
    ENDIF

    testq=where(data_sed.sigmaQQ NE 0. and finite(data_sed.sigmaQQ) EQ 1 and data_sed.sigmaQQ NE la_undef() and data_sed.StokesQ NE la_undef() and finite(data_sed.StokesQ) EQ 1,ctestq)                

    IF ctestq NE 0 THEN BEGIN 

        instru_names = qsed.instru_names(testq)
        filt_names = qsed.filt_names(testq)
        wav = qsed.wav(testq)
        values = qsed.values(testq)
        sigma = qsed.sigma(testq)
        qsed={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    ENDIF
    ENDELSE
    uzone:
    ans=''

    used.values=data_sed.StokesU
    used.sigma=sqrt(abs(data_sed.sigmaUU))

    ;Testig if there is StokesU data. If not return empty value and end loop.

    ind = where(data_sed.StokesU NE la_undef() and finite(data_sed.StokesU) EQ 1 and data_sed.sigmaUU NE la_undef() and finite(data_sed.sigmaUU) EQ 1,NU)

    IF NU EQ 0 THEN BEGIN

        used = ptr_new()

        goto, psiemzone

    ENDIF ELSE BEGIN

    ind = where(finite(data_sed.StokesU) EQ 0 or finite(data_sed.sigmaUU) EQ 0, countnoku) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 

    IF countnoku NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in StokesU data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue

            message, 'Aborting ...'

        ENDIF
    ENDIF  
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue

    tdaerru=where((data_sed.sigmaUU EQ la_undef() and data_sed.StokesU NE la_undef()) or (data_sed.sigmaUU NE la_undef() and data_sed.StokesU EQ la_undef()),ctdaerru)

    IF ctdaerru NE 0 THEN BEGIN
        message, 'Data and errors in Stokes U do not match. Undefinded data detected for these filters:',/continue
        print,data_sed[tdaerru].filter
        stop
    ENDIF

    testu=where(data_sed.sigmaUU NE 0. and finite(data_sed.sigmaUU) EQ 1 and data_sed.sigmaUU NE la_undef() and data_sed.StokesU NE la_undef() and finite(data_sed.StokesU) EQ 1,ctestu)                

    IF ctestu NE 0 THEN BEGIN 

        instru_names = used.instru_names(testu)
        filt_names = used.filt_names(testu)
        wav = used.wav(testu)
        values = used.values(testu)
        sigma = used.sigma(testu)
        used={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    ENDIF
    ENDELSE

    psiemzone:
    

    if nu eq 0 and nq eq 0 then goto, end_sed

    degtorad = !pi/180
    

    ;Generation of POLFRAC,POLSED and PSI data by default   

    ;Testing on the presence of covariances otherwise an error is returned

    ;The testing happens here because covariances are needed for the error estimation of P,p and PSI.   

     
    tstcovu = where((data_sed(testu).sigmaIU) EQ la_undef() or (data_sed(testu).sigmaQU) EQ la_undef(), ctcovu)

    if ctcovu NE 0  then BEGIN
        ;message, 'Covariance data is missing in StokesU data. Data, errors and covariances do not match. Aborting...',/continue
        message, 'IU or QU Covariance data is missing in StokesU data. Data, errors and covariances do not match for the following filters.',/continue
        print,data_sed[testu[tstcovu]].filter
        stop
    ENDIF

    tstcovq = where((data_sed(testq).sigmaIQ) EQ la_undef() or (data_sed(testq).sigmaQU) EQ la_undef(), ctcovq)

    if ctcovq NE 0  then BEGIN
        message, 'IQ or QU Covariance data is missing in StokesQ data. Data, errors and covariances do not match for the following filters.',/continue
        print,data_sed[testq[tstcovq]].filter
        stop
    ENDIF

    ;Matching both filter and spectrum points 
    
    match2,used.wav,qsed.wav,qpts,upts
    
    ;removing -1 values from the indices (qpts & upts)
    indnotq = where(qpts eq -1, cnotq)
    if cnotq ne 0 then remove, indnotq, qpts
    indnotu = where(upts eq -1, cnotu)
    if cnotu ne 0 then remove, indnotu, upts
    
    match2, sed.wav, qsed.wav(qpts), qtmp, sedpts
    indnoti = where(sedpts eq -1, cnoti)
    if cnoti ne 0 then remove, indnoti, sedpts
    
    instru_names = used.instru_names(upts)
    filt_names = used.filt_names(upts)
    wav = used.wav(upts)
    values = 0.5*atan(used.values(upts),qsed.values(qpts))/degtorad
    polar_variance_iqu2ippsi,sed.values(sedpts),qsed.values(qpts),used.values(upts),(data_sed(testi).sigmaII)(sedpts),(data_sed(testq).sigmaQQ)(qpts),(data_sed(testu).sigmaUU)(upts),(data_sed(testi).sigmaIQ)(sedpts),(data_sed(testi).sigmaIU)(sedpts),(data_sed(testi).sigmaQU)(sedpts),variance_smallp,variance_psi,variance_largep
    sigma = sqrt(variance_psi)
    psi_em={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
    

    polsed = psi_em
    polsed.values = sqrt(used.values(upts)^2+qsed.values(qpts)^2) 
    polsed.sigma = sqrt(variance_largep)

    
    polfrac = psi_em
    polfrac.values = polsed.values/sed.values(sedpts) 
    polfrac.sigma = sqrt(variance_smallp)
    
    goto, end_sed

    end_sed:

ENDIF

;EXTINCTION HASN'T BEEN TREATED YET 

;SMALLL ISSUE .. POLEXTFRAC is not present. Let's create it for completness

IF KEYWORD_SET(data_ext) THEN begin
    

    ans=''

    wavs=data_ext.wave

    ;=== Impose central wavelengths for photometric channels

    ind=where(data_ext.filter NE 'SPECTRUM',count)

    IF count NE 0 THEN BEGIN

        wavs(ind)=dustem_filter2wav(data_ext(ind).filter)

    ENDIF
    ;=== define observations structure 

    obs_EXT={instru_names:data_ext.instru,filt_names:data_ext.filter,wav:wavs, values:data_ext.EXT_I,sigma:sqrt(abs(data_ext.sigextII))} 

    ;INITIALIZING OUTPUT STRUCTURES

    ext = obs_EXT

    qext = ext
    uext = ext

    
    ind = where(data_ext.EXT_I NE la_undef() and finite(data_ext.EXT_I) EQ 1 and data_ext.sigextII NE la_undef() and finite(data_ext.sigextII) EQ 1,N_ext) ;Testing if the data and errors are set

    IF N_ext EQ 0 THEN BEGIN

        ext = ptr_new()

        goto, qexzone

    ENDIF ELSE BEGIN

    ind = where(finite(data_ext.EXT_I) EQ 0 or finite(data_ext.sigextII) EQ 0, countnokex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 

    IF countnokex NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in EXT_I data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue

            message, 'Aborting ...'

        ENDIF
    ENDIF      
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $ 
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue

    tdaerrex=where((data_ext.sigextII EQ la_undef() and data_ext.EXT_I NE la_undef()) or (data_ext.sigextII NE la_undef() and data_ext.EXT_I EQ la_undef()),ctdaerrex)
    IF ctdaerrex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testex=where(data_ext.sigextII NE 0. and finite(data_ext.sigextII) EQ 1 and data_ext.sigextII NE la_undef() and data_ext.EXT_I NE la_undef() and finite(data_ext.EXT_I) EQ 1,ctestex)                

    IF ctestex NE 0 THEN BEGIN 

        instru_names = ext.instru_names(testex)
        filt_names = ext.filt_names(testex)
        wav = ext.wav(testex)
        values = ext.values(testex)
        sigma = ext.sigma(testex)
        ext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    ENDIF 
    ENDELSE

    qexzone:
    ans=''

    qext.values=data_ext.EXT_Q
    qext.sigma=sqrt(abs(data_ext.sigextQQ))

    ;Testig if there is EXT_P data. If not move to EXT_U testing

    ind = where(data_ext.EXT_Q NE la_undef() and finite(data_ext.EXT_Q) EQ 1 and data_ext.sigextQQ NE la_undef() and finite(data_ext.sigextQQ) EQ 1,NQ_ext) ;Testing if the data and errors are set

    ;dustem_set_data.prostop   

    IF NQ_ext EQ 0 THEN BEGIN

        qext = ptr_new()

        goto, uexzone
    ENDIF ELSE BEGIN

    ind = where(finite(data_ext.EXT_Q) EQ 0 or finite(data_ext.sigextQQ) EQ 0, countnokqex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 

    IF countnokqex NE 0 THEN BEGIN 

        message, "SED format isn't met. NaN value(s) detected in EXT_Q data.",/continue

        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue

            message, 'Aborting ...'

        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue

    tdaerrqex=where((data_ext.sigextQQ EQ la_undef() and data_ext.EXT_Q NE la_undef()) or (data_ext.sigextQQ NE la_undef() and data_ext.EXT_Q EQ la_undef()),ctdaerrqex)
    IF ctdaerrqex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testqex=where(data_ext.sigextQQ NE 0. and finite(data_ext.sigextQQ) EQ 1 and data_ext.sigextQQ NE la_undef() and data_ext.EXT_Q NE la_undef() and finite(data_ext.EXT_Q) EQ 1,ctestqex)                

    IF ctestqex NE 0 THEN BEGIN 

        instru_names = qext.instru_names(testqex)
        filt_names = qext.filt_names(testqex)
        wav = qext.wav(testqex)
        values = qext.values(testqex)
        sigma = qext.sigma(testqex)
        qext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    ENDIF 
    ENDELSE

    uexzone:
    ans=''

    uext.values=data_ext.EXT_U
    uext.sigma=sqrt(abs(data_ext.sigextUU))

    ;Testig if there is EXT_P data. If not return empty value and end loop.

    ind = where(data_ext.EXT_U NE la_undef() and finite(data_ext.EXT_U) EQ 1 and data_ext.sigextUU NE la_undef() and finite(data_ext.sigextUU) EQ 1,NU_ext) ;Testing if the data and errors are set

    IF NU_ext EQ 0 THEN BEGIN

        uext = ptr_new()

        goto, psiexzone

    ENDIF ELSE BEGIN

    ind = where(finite(data_ext.EXT_U) EQ 0 or finite(data_ext.sigextUU) EQ 0, countnokuex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 

    IF countnokuex NE 0 THEN BEGIN 

        message, "SED format isn't met. NaN value(s) detected in EXT_U data.",/continue

        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue

            message, 'Aborting ...'

        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue

    tdaerruex=where((data_ext.sigextUU EQ la_undef() and data_ext.EXT_U NE la_undef()) or (data_ext.sigextUU NE la_undef() and data_ext.EXT_U EQ la_undef()),ctdaerruex)
    IF ctdaerruex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testuex=where(data_ext.sigextUU NE 0. and finite(data_ext.sigextUU) EQ 1 and data_ext.sigextUU NE la_undef() and data_ext.EXT_U NE la_undef() and finite(data_ext.EXT_U) EQ 1,ctestuex)                

    IF ctestuex NE 0 THEN BEGIN 

        instru_names = uext.instru_names(testuex)
        filt_names = uext.filt_names(testuex)
        wav = uext.wav(testuex)
        values = uext.values(testuex)
        sigma = uext.sigma(testuex)
        uext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    ENDIF 
    ENDELSE

    
    psiexzone: ;Starting the analogy here. 
    

    if NU_EXT eq 0 and NQ_EXT eq 0 then goto, end_ext

    degtorad = !pi/180
    
    ;Generation of POLEXT and PSI_EXT data by default   
    ;Testing on the presence of covariances otherwise an error is returned
    ;The testing happens here because covariances are needed for the error estimation of EXT_P,PSI_EXT.   
    

    tstcovuex = where((data_ext(testuex).sigextIU) EQ la_undef() or (data_ext(testuex).sigextQU) EQ la_undef(), ctcovuex)

    if ctcovuex NE 0  then message, 'Covariance data is missing in the extinction data pertaining to the StokesU parameter. Data, errors and covariances do not match. Aborting...'
    tstcovqex = where((data_ext(testqex).sigextIQ) EQ la_undef() or (data_ext(testqex).sigextQU) EQ la_undef(), ctcovqex)
    if ctcovqex NE 0  then message, 'Covariance data is missing in the extinction pertaining to the StokesQ parameter. Data, errors and covariances do not match. Aborting...'
    
    ;Matching both filter and spectrum points 
    match2,uext.wav,qext.wav,qexpts,uexpts
    
    ;removing -1 values from the indices (qpts & upts)
    indnotqex = where(qexpts eq -1, cnotqex)
    if cnotqex ne 0 then remove, indnotqex, qexpts
    indnotuex = where(uexpts eq -1, cnotuex)
    if cnotuex ne 0 then remove, indnotuex, uexpts
    

    match2, ext.wav, qext.wav(qexpts), qextmp, extpts

    indnotx = where(extpts eq -1, cnotx)
    if cnotx ne 0 then remove, indnotx, extpts
    
    instru_names = uext.instru_names(uexpts)
    filt_names = uext.filt_names(uexpts)
    wav = uext.wav(uexpts)
    values = 0.5*atan(uext.values(uexpts),qext.values(qexpts))/degtorad

    polar_variance_iqu2ippsi,ext.values(extpts),qext.values(qexpts),uext.values(uexpts),(data_ext(testex).sigextII)(extpts),(data_ext(testqex).sigextQQ)(qexpts),(data_ext(testuex).sigextUU)(uexpts),(data_ext(testex).sigextIQ)(extpts),(data_ext(testex).sigextIU)(extpts),(data_ext(testex).sigextQU)(extpts),variance_smallp,variance_psi,variance_largep

    sigma = sqrt(variance_psi)

    psi_ext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}

    polext = psi_ext

    polext.values = sqrt(uext.values(uexpts)^2+qext.values(qexpts)^2) 

    polext.sigma = sqrt(variance_largep)

    
    fpolext = psi_ext
    fpolext.values = polext.values/ext.values(extpts) 
    fpolext.sigma = sqrt(variance_smallp)
    

    goto, end_ext
    

    end_ext:
ENDIF

the_end:

END