Blame view

src/idl/dustem_check_data.pro 21.9 KB
88872c38   Ilyes Choubani   small update. Cha...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
PRO dustem_check_data, data_sed, $
                       data_ext, $
                       sed=sed, $
                       ext=ext, $
                       polext=polext, $
                       polsed=polsed, $
                       polfrac=polfrac, $
                       psi_em=psi_em, $
                       qsed=qsed, $
                       used=used, $
                       qext=qext, $
                       uext=uext, $
                       psi_ext=psi_ext, $
                       fpolext=fpolext
09e61d64   Ilyes Choubani   cleaned set_data ...
15

04efbaa5   Ilyes Choubani   Correcting set_da...
16
17
;+
; NAME:
edeb9236   Annie Hughes   updated doc help
18
19
;    dustem_check_data
;
04efbaa5   Ilyes Choubani   Correcting set_da...
20
; PURPOSE:
88872c38   Ilyes Choubani   small update. Cha...
21
22
;    Clean and modify the input DustEMWrap structure to meet the requirements of DustEMWrap. 
;    This is important for the subsequent internal handling/fitting of data. 
edeb9236   Annie Hughes   updated doc help
23
;
04efbaa5   Ilyes Choubani   Correcting set_da...
24
; CATEGORY:
edeb9236   Annie Hughes   updated doc help
25
26
;    DustEMWrap, Mid-Level, Distributed, Initialization
;
04efbaa5   Ilyes Choubani   Correcting set_da...
27
; CALLING SEQUENCE:
88872c38   Ilyes Choubani   small update. Cha...
28
;    dustem_check_data,data_sed,data_ext,sed=sed,ext=ext,polext=polext,polsed=polsed,polfrac=polfrac,psi_em=psi_em,qsed=qsed,used=used,qext=qext,uext=uext,psi_ext=psi_ext,fpolext=fpolext,[/help]
edeb9236   Annie Hughes   updated doc help
29
;
04efbaa5   Ilyes Choubani   Correcting set_da...
30
; INPUTS:
88872c38   Ilyes Choubani   small update. Cha...
31
32
;    data_sed : Emission DustEMWrap structure
;    data_ext : Extinction DustEMWrap structure
edeb9236   Annie Hughes   updated doc help
33
;
04efbaa5   Ilyes Choubani   Correcting set_da...
34
35
; OPTIONAL INPUT PARAMETERS:
;    None
edeb9236   Annie Hughes   updated doc help
36
;
88872c38   Ilyes Choubani   small update. Cha...
37
38
39
40
41
42
43
44
45
46
47
48
49
; OUTPUTS: 
;    sed : modified emission structure 
;    ext : modified extinction structure
;    polext : modified polarized extinction structure
;    polsed : modified polarized emission structure 
;    polfrac : modified fractional polarized emission structure
;    psi_em  : modified emission polarization angle structure 
;    qsed : modified StokesQ emission structure
;    used : modified StokesU emission structure
;    qext : modified StokesQ extinction structure
;    uext : modified StokesU extinction structure
;    psi_ext : modified extinction polarization angle structure
;    fpolext : modified fractional extinction polarization structure
edeb9236   Annie Hughes   updated doc help
50
;
04efbaa5   Ilyes Choubani   Correcting set_da...
51
52
; OPTIONAL OUTPUT PARAMETERS:
;    None
edeb9236   Annie Hughes   updated doc help
53
;
04efbaa5   Ilyes Choubani   Correcting set_da...
54
55
; ACCEPTED KEY-WORDS:
;    help      = If set print this help
edeb9236   Annie Hughes   updated doc help
56
;
04efbaa5   Ilyes Choubani   Correcting set_da...
57
58
; COMMON BLOCKS:
;    None
edeb9236   Annie Hughes   updated doc help
59
;
04efbaa5   Ilyes Choubani   Correcting set_da...
60
; SIDE EFFECTS:
edeb9236   Annie Hughes   updated doc help
61
;
04efbaa5   Ilyes Choubani   Correcting set_da...
62
; RESTRICTIONS:
edeb9236   Annie Hughes   updated doc help
63
64
;    The DustEMWrap IDL code must be installed
;
04efbaa5   Ilyes Choubani   Correcting set_da...
65
66
; PROCEDURE:
;    None
edeb9236   Annie Hughes   updated doc help
67
;
04efbaa5   Ilyes Choubani   Correcting set_da...
68
; EXAMPLES
88872c38   Ilyes Choubani   small update. Cha...
69
70
;    dustem_check_data,data_sed,data_ext,sed=sed,ext=ext,polext=polext,polsed=polsed,polfrac=polfrac,psi_em=psi_em,qsed=qsed,used=used,qext=qext,uext=uext,psi_ext=psi_ext,fpolext=fpolext,[/help]
;    dustem_check_data,/help
edeb9236   Annie Hughes   updated doc help
71
; MODIFICATION HISTORY:
88872c38   Ilyes Choubani   small update. Cha...
72
;    Written IC 2022
edeb9236   Annie Hughes   updated doc help
73
74
;    Evolution details on the DustEMWrap gitlab.
;    See http://dustemwrap.irap.omp.eu/ for FAQ and help.   
88872c38   Ilyes Choubani   small update. Cha...
75
76
77
; COMMENTS:
;    Might still need a bit of cleaning. I am unsure of the availability of covariance data in polarized extinction.
;    Still regarding extinction, some of the error messages haven't been mirrored correctly with respect to the emission block.
04efbaa5   Ilyes Choubani   Correcting set_da...
78
79
80
81
82
83
84
85
;-


IF keyword_set(help) THEN BEGIN
  doc_library,'dustem_check_data'
  goto,the_end
ENDIF

006bb639   Jean-Philippe Bernard   improved erro mes...
86

09e61d64   Ilyes Choubani   cleaned set_data ...
87
;CHECKING THE FORMAT OF THE SED STRUCTURE
3ff9cac3   Ilyes Choubani   general update
88
89
IF KEYWORD_SET(data_sed) THEN BEGIN
            
09e61d64   Ilyes Choubani   cleaned set_data ...
90
    ans=''
3ff9cac3   Ilyes Choubani   general update
91
    wavs=data_sed.wave
09e61d64   Ilyes Choubani   cleaned set_data ...
92
    ;=== Impose central wavelengths for photometric channels
3ff9cac3   Ilyes Choubani   general update
93
    ind=where(data_sed.filter NE 'SPECTRUM',count)
09e61d64   Ilyes Choubani   cleaned set_data ...
94
    IF count NE 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
95
        wavs(ind)=dustem_filter2wav(data_sed(ind).filter)
09e61d64   Ilyes Choubani   cleaned set_data ...
96
    ENDIF
3ff9cac3   Ilyes Choubani   general update
97
    ;=== Initializing observation structure
3ff9cac3   Ilyes Choubani   general update
98
    obs_EM={instru_names:data_sed.instru,filt_names:data_sed.filter,wav:wavs, values:data_sed.StokesI,sigma:sqrt(abs(data_sed.sigmaII))} 
09e61d64   Ilyes Choubani   cleaned set_data ...
99
    ;INITIALIZING OUTPUT STRUCTURES
3ff9cac3   Ilyes Choubani   general update
100
    sed = obs_EM
3ff9cac3   Ilyes Choubani   general update
101
102
    qsed = sed
    used = sed
3ff9cac3   Ilyes Choubani   general update
103
    
09e61d64   Ilyes Choubani   cleaned set_data ...
104
    ;Test if there is StokesI data. If not move to polsed testing.
3ff9cac3   Ilyes Choubani   general update
105
    ind = where(data_sed.StokesI NE la_undef() and finite(data_sed.StokesI) EQ 1 and data_sed.sigmaII NE la_undef() and finite(data_sed.sigmaII) EQ 1,N_sed) ;Testing if the data and errors are set
09e61d64   Ilyes Choubani   cleaned set_data ...
106
    IF N_sed EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
107
        sed = ptr_new()
abebbdf3   Ilyes Choubani   General update: A...
108
        goto, qzone
09e61d64   Ilyes Choubani   cleaned set_data ...
109
    ENDIF ELSE BEGIN 
006bb639   Jean-Philippe Bernard   improved erro mes...
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
        ind = where(finite(data_sed.StokesI) EQ 0 or finite(data_sed.sigmaII) EQ 0, countnoki) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
        IF countnoki NE 0 THEN BEGIN
            message, "SED format isn't met. NaN value(s) detected in StokesI data.",/continue
            message, 'Without automatic format fitering you cannot proceed.',/continue
            ;stop
            read, ans, prompt='Would you like automatic format filtering? (Y/N):'
            IF strupcase(ans) EQ 'N' THEN BEGIN
                message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
                message, 'Aborting ...'
            ENDIF
        ENDIF 
        IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
        ELSE message, 'Data format is met. Removal of undefined data points...',/continue
        tdaerri=where((data_sed.sigmaII EQ la_undef() and data_sed.StokesI NE la_undef()) or (data_sed.sigmaII NE la_undef() and data_sed.StokesI EQ la_undef()),ctdaerri)
        IF ctdaerri NE 0 THEN BEGIN
            message, 'Data and errors do not match. Undefinded data detected. Aborting...',/continue
09e61d64   Ilyes Choubani   cleaned set_data ...
126
        ENDIF
006bb639   Jean-Philippe Bernard   improved erro mes...
127
128
129
130
131
132
133
134
135
        testi=where(data_sed.sigmaII NE 0. and finite(data_sed.sigmaII) EQ 1 and data_sed.sigmaII NE la_undef() and data_sed.StokesI NE la_undef() and finite(data_sed.StokesI) EQ 1,ctesti)                
        IF ctesti NE 0 THEN BEGIN 
            instru_names = sed.instru_names(testi)
            filt_names = sed.filt_names(testi)
            wav = sed.wav(testi)
            values = sed.values(testi)
            sigma = sed.sigma(testi)
            sed={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
        ENDIF    
09e61d64   Ilyes Choubani   cleaned set_data ...
136
    ENDELSE
09e61d64   Ilyes Choubani   cleaned set_data ...
137
138
    qzone:
    ans=''
3ff9cac3   Ilyes Choubani   general update
139
140
    qsed.values=data_sed.StokesQ
    qsed.sigma=sqrt(abs(data_sed.sigmaQQ))
09e61d64   Ilyes Choubani   cleaned set_data ...
141
    ;Testig if there is StokesQ data. If not move to used data
3ff9cac3   Ilyes Choubani   general update
142
    ind = where(data_sed.StokesQ NE la_undef() and finite(data_sed.StokesQ) EQ 1 and data_sed.sigmaQQ NE la_undef() and finite(data_sed.sigmaQQ) EQ 1,NQ)
09e61d64   Ilyes Choubani   cleaned set_data ...
143
    IF NQ EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
144
        qsed = ptr_new()
09e61d64   Ilyes Choubani   cleaned set_data ...
145
146
        goto, uzone
    ENDIF ELSE BEGIN
3ff9cac3   Ilyes Choubani   general update
147
    ind = where(finite(data_sed.StokesQ) EQ 0 or finite(data_sed.sigmaQQ) EQ 0, countnokq) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
09e61d64   Ilyes Choubani   cleaned set_data ...
148
149
150
151
152
153
    IF countnokq NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in StokesQ data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
266ae799   Ilyes Choubani   General update
154
            message, 'Aborting ...'
09e61d64   Ilyes Choubani   cleaned set_data ...
155
156
157
158
        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue
3ff9cac3   Ilyes Choubani   general update
159
    tdaerrq=where((data_sed.sigmaQQ EQ la_undef() and data_sed.StokesQ NE la_undef()) or (data_sed.sigmaQQ NE la_undef() and data_sed.StokesQ EQ la_undef()),ctdaerrq)
006bb639   Jean-Philippe Bernard   improved erro mes...
160
161
162
163
164
165
    IF ctdaerrq NE 0 THEN BEGIN
        message, 'Data and errors in Stokes Q do not match. Undefinded data detected for these filters:',/continue
        print,data_sed[tdaerrq].filter
        ;message, 'Data and errors do not match. Undefinded data detected. Aborting...',/continue
        stop
    ENDIF
3ff9cac3   Ilyes Choubani   general update
166
    testq=where(data_sed.sigmaQQ NE 0. and finite(data_sed.sigmaQQ) EQ 1 and data_sed.sigmaQQ NE la_undef() and data_sed.StokesQ NE la_undef() and finite(data_sed.StokesQ) EQ 1,ctestq)                
09e61d64   Ilyes Choubani   cleaned set_data ...
167
    IF ctestq NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
168
169
170
171
172
173
        instru_names = qsed.instru_names(testq)
        filt_names = qsed.filt_names(testq)
        wav = qsed.wav(testq)
        values = qsed.values(testq)
        sigma = qsed.sigma(testq)
        qsed={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
09e61d64   Ilyes Choubani   cleaned set_data ...
174
175
176
177
    ENDIF
    ENDELSE
    uzone:
    ans=''
3ff9cac3   Ilyes Choubani   general update
178
179
    used.values=data_sed.StokesU
    used.sigma=sqrt(abs(data_sed.sigmaUU))
09e61d64   Ilyes Choubani   cleaned set_data ...
180
    ;Testig if there is StokesU data. If not return empty value and end loop.
3ff9cac3   Ilyes Choubani   general update
181
    ind = where(data_sed.StokesU NE la_undef() and finite(data_sed.StokesU) EQ 1 and data_sed.sigmaUU NE la_undef() and finite(data_sed.sigmaUU) EQ 1,NU)
09e61d64   Ilyes Choubani   cleaned set_data ...
182
    IF NU EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
183
        used = ptr_new()
abebbdf3   Ilyes Choubani   General update: A...
184
        goto, psiemzone
09e61d64   Ilyes Choubani   cleaned set_data ...
185
    ENDIF ELSE BEGIN
3ff9cac3   Ilyes Choubani   general update
186
    ind = where(finite(data_sed.StokesU) EQ 0 or finite(data_sed.sigmaUU) EQ 0, countnoku) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
09e61d64   Ilyes Choubani   cleaned set_data ...
187
188
189
190
191
192
    IF countnoku NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in StokesU data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
266ae799   Ilyes Choubani   General update
193
            message, 'Aborting ...'
09e61d64   Ilyes Choubani   cleaned set_data ...
194
195
196
197
        ENDIF
    ENDIF  
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue
3ff9cac3   Ilyes Choubani   general update
198
    tdaerru=where((data_sed.sigmaUU EQ la_undef() and data_sed.StokesU NE la_undef()) or (data_sed.sigmaUU NE la_undef() and data_sed.StokesU EQ la_undef()),ctdaerru)
006bb639   Jean-Philippe Bernard   improved erro mes...
199
200
201
202
203
    IF ctdaerru NE 0 THEN BEGIN
        message, 'Data and errors in Stokes U do not match. Undefinded data detected for these filters:',/continue
        print,data_sed[tdaerru].filter
        stop
    ENDIF
3ff9cac3   Ilyes Choubani   general update
204
    testu=where(data_sed.sigmaUU NE 0. and finite(data_sed.sigmaUU) EQ 1 and data_sed.sigmaUU NE la_undef() and data_sed.StokesU NE la_undef() and finite(data_sed.StokesU) EQ 1,ctestu)                
09e61d64   Ilyes Choubani   cleaned set_data ...
205
    IF ctestu NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
206
207
208
209
210
211
        instru_names = used.instru_names(testu)
        filt_names = used.filt_names(testu)
        wav = used.wav(testu)
        values = used.values(testu)
        sigma = used.sigma(testu)
        used={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
09e61d64   Ilyes Choubani   cleaned set_data ...
212
213
    ENDIF
    ENDELSE
abebbdf3   Ilyes Choubani   General update: A...
214
215
    psiemzone:
    
303de20d   Ilyes Choubani   updating check_data
216
    if nu eq 0 and nq eq 0 then goto, end_sed
abebbdf3   Ilyes Choubani   General update: A...
217
218
    degtorad = !pi/180
    
303de20d   Ilyes Choubani   updating check_data
219
    ;Generation of POLFRAC,POLSED and PSI data by default   
abebbdf3   Ilyes Choubani   General update: A...
220
    ;Testing on the presence of covariances otherwise an error is returned
88872c38   Ilyes Choubani   small update. Cha...
221
    ;The testing happens here because covariances are needed for the error estimation of P,p and PSI. (Also cannot test before because we're not in !run_pol mode) 
abebbdf3   Ilyes Choubani   General update: A...
222
    tstcovu = where((data_sed(testu).sigmaIU) EQ la_undef() or (data_sed(testu).sigmaQU) EQ la_undef(), ctcovu)
006bb639   Jean-Philippe Bernard   improved erro mes...
223
224
225
226
227
228
    if ctcovu NE 0  then BEGIN
        ;message, 'Covariance data is missing in StokesU data. Data, errors and covariances do not match. Aborting...',/continue
        message, 'IU or QU Covariance data is missing in StokesU data. Data, errors and covariances do not match for the following filters.',/continue
        print,data_sed[testu[tstcovu]].filter
        stop
    ENDIF
abebbdf3   Ilyes Choubani   General update: A...
229
    tstcovq = where((data_sed(testq).sigmaIQ) EQ la_undef() or (data_sed(testq).sigmaQU) EQ la_undef(), ctcovq)
006bb639   Jean-Philippe Bernard   improved erro mes...
230
231
232
233
234
    if ctcovq NE 0  then BEGIN
        message, 'IQ or QU Covariance data is missing in StokesQ data. Data, errors and covariances do not match for the following filters.',/continue
        print,data_sed[testq[tstcovq]].filter
        stop
    ENDIF
abebbdf3   Ilyes Choubani   General update: A...
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
    ;Matching both filter and spectrum points 
    
    match2,used.wav,qsed.wav,qpts,upts
    
    ;removing -1 values from the indices (qpts & upts)
    indnotq = where(qpts eq -1, cnotq)
    if cnotq ne 0 then remove, indnotq, qpts
    indnotu = where(upts eq -1, cnotu)
    if cnotu ne 0 then remove, indnotu, upts
    
    match2, sed.wav, qsed.wav(qpts), qtmp, sedpts
    indnoti = where(sedpts eq -1, cnoti)
    if cnoti ne 0 then remove, indnoti, sedpts
    
    instru_names = used.instru_names(upts)
    filt_names = used.filt_names(upts)
    wav = used.wav(upts)
    values = 0.5*atan(used.values(upts),qsed.values(qpts))/degtorad
    polar_variance_iqu2ippsi,sed.values(sedpts),qsed.values(qpts),used.values(upts),(data_sed(testi).sigmaII)(sedpts),(data_sed(testq).sigmaQQ)(qpts),(data_sed(testu).sigmaUU)(upts),(data_sed(testi).sigmaIQ)(sedpts),(data_sed(testi).sigmaIU)(sedpts),(data_sed(testi).sigmaQU)(sedpts),variance_smallp,variance_psi,variance_largep
    sigma = sqrt(variance_psi)
    psi_em={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
    
abebbdf3   Ilyes Choubani   General update: A...
257
258
259
    polsed = psi_em
    polsed.values = sqrt(used.values(upts)^2+qsed.values(qpts)^2) 
    polsed.sigma = sqrt(variance_largep)
abebbdf3   Ilyes Choubani   General update: A...
260
261
262
263
264
265
    
    polfrac = psi_em
    polfrac.values = polsed.values/sed.values(sedpts) 
    polfrac.sigma = sqrt(variance_smallp)
    
    goto, end_sed
09e61d64   Ilyes Choubani   cleaned set_data ...
266
    end_sed:
303de20d   Ilyes Choubani   updating check_data
267
    
09e61d64   Ilyes Choubani   cleaned set_data ...
268
269
270
ENDIF


88872c38   Ilyes Choubani   small update. Cha...
271
272
273
;EXTINCTION
;
;Covariances?... 
3ff9cac3   Ilyes Choubani   general update
274
275
IF KEYWORD_SET(data_ext) THEN begin
    
09e61d64   Ilyes Choubani   cleaned set_data ...
276
    ans=''
3ff9cac3   Ilyes Choubani   general update
277
    wavs=data_ext.wave
09e61d64   Ilyes Choubani   cleaned set_data ...
278
    ;=== Impose central wavelengths for photometric channels
3ff9cac3   Ilyes Choubani   general update
279
    ind=where(data_ext.filter NE 'SPECTRUM',count)
09e61d64   Ilyes Choubani   cleaned set_data ...
280
    IF count NE 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
281
        wavs(ind)=dustem_filter2wav(data_ext(ind).filter)
09e61d64   Ilyes Choubani   cleaned set_data ...
282
283
    ENDIF
    ;=== define observations structure 
3ff9cac3   Ilyes Choubani   general update
284
    obs_EXT={instru_names:data_ext.instru,filt_names:data_ext.filter,wav:wavs, values:data_ext.EXT_I,sigma:sqrt(abs(data_ext.sigextII))} 
09e61d64   Ilyes Choubani   cleaned set_data ...
285
    ;INITIALIZING OUTPUT STRUCTURES
3ff9cac3   Ilyes Choubani   general update
286
    ext = obs_EXT
3ff9cac3   Ilyes Choubani   general update
287
288
    qext = ext
    uext = ext
3ff9cac3   Ilyes Choubani   general update
289
290
    
    ind = where(data_ext.EXT_I NE la_undef() and finite(data_ext.EXT_I) EQ 1 and data_ext.sigextII NE la_undef() and finite(data_ext.sigextII) EQ 1,N_ext) ;Testing if the data and errors are set
09e61d64   Ilyes Choubani   cleaned set_data ...
291
    IF N_ext EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
292
        ext = ptr_new()
303de20d   Ilyes Choubani   updating check_data
293
        goto, qexzone
09e61d64   Ilyes Choubani   cleaned set_data ...
294
    ENDIF ELSE BEGIN
3ff9cac3   Ilyes Choubani   general update
295
    ind = where(finite(data_ext.EXT_I) EQ 0 or finite(data_ext.sigextII) EQ 0, countnokex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
09e61d64   Ilyes Choubani   cleaned set_data ...
296
297
298
299
300
301
302
    IF countnokex NE 0 THEN BEGIN 
        message, "SED format isn't met. NaN value(s) detected in EXT_I data.",/continue
        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
266ae799   Ilyes Choubani   General update
303
            message, 'Aborting ...'
09e61d64   Ilyes Choubani   cleaned set_data ...
304
305
306
307
        ENDIF
    ENDIF      
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $ 
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue
3ff9cac3   Ilyes Choubani   general update
308
309
310
    tdaerrex=where((data_ext.sigextII EQ la_undef() and data_ext.EXT_I NE la_undef()) or (data_ext.sigextII NE la_undef() and data_ext.EXT_I EQ la_undef()),ctdaerrex)
    IF ctdaerrex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testex=where(data_ext.sigextII NE 0. and finite(data_ext.sigextII) EQ 1 and data_ext.sigextII NE la_undef() and data_ext.EXT_I NE la_undef() and finite(data_ext.EXT_I) EQ 1,ctestex)                
09e61d64   Ilyes Choubani   cleaned set_data ...
311
    IF ctestex NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
312
313
314
315
316
317
        instru_names = ext.instru_names(testex)
        filt_names = ext.filt_names(testex)
        wav = ext.wav(testex)
        values = ext.values(testex)
        sigma = ext.sigma(testex)
        ext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
09e61d64   Ilyes Choubani   cleaned set_data ...
318
319
    ENDIF 
    ENDELSE
303de20d   Ilyes Choubani   updating check_data
320
    
09e61d64   Ilyes Choubani   cleaned set_data ...
321
322
    qexzone:
    ans=''
3ff9cac3   Ilyes Choubani   general update
323
324
    qext.values=data_ext.EXT_Q
    qext.sigma=sqrt(abs(data_ext.sigextQQ))
09e61d64   Ilyes Choubani   cleaned set_data ...
325
    ;Testig if there is EXT_P data. If not move to EXT_U testing
3ff9cac3   Ilyes Choubani   general update
326
    ind = where(data_ext.EXT_Q NE la_undef() and finite(data_ext.EXT_Q) EQ 1 and data_ext.sigextQQ NE la_undef() and finite(data_ext.sigextQQ) EQ 1,NQ_ext) ;Testing if the data and errors are set
9ee42e40   Ilyes Choubani   fixed small bugs
327
    ;dustem_set_data.prostop   
09e61d64   Ilyes Choubani   cleaned set_data ...
328
    IF NQ_ext EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
329
        qext = ptr_new()
09e61d64   Ilyes Choubani   cleaned set_data ...
330
331
        goto, uexzone
    ENDIF ELSE BEGIN
3ff9cac3   Ilyes Choubani   general update
332
    ind = where(finite(data_ext.EXT_Q) EQ 0 or finite(data_ext.sigextQQ) EQ 0, countnokqex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
09e61d64   Ilyes Choubani   cleaned set_data ...
333
    IF countnokqex NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
334
        message, "SED format isn't met. NaN value(s) detected in EXT_Q data.",/continue
09e61d64   Ilyes Choubani   cleaned set_data ...
335
336
337
338
339
        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
266ae799   Ilyes Choubani   General update
340
            message, 'Aborting ...'
09e61d64   Ilyes Choubani   cleaned set_data ...
341
342
343
344
        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue
3ff9cac3   Ilyes Choubani   general update
345
346
347
    tdaerrqex=where((data_ext.sigextQQ EQ la_undef() and data_ext.EXT_Q NE la_undef()) or (data_ext.sigextQQ NE la_undef() and data_ext.EXT_Q EQ la_undef()),ctdaerrqex)
    IF ctdaerrqex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testqex=where(data_ext.sigextQQ NE 0. and finite(data_ext.sigextQQ) EQ 1 and data_ext.sigextQQ NE la_undef() and data_ext.EXT_Q NE la_undef() and finite(data_ext.EXT_Q) EQ 1,ctestqex)                
09e61d64   Ilyes Choubani   cleaned set_data ...
348
    IF ctestqex NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
349
350
351
352
353
354
        instru_names = qext.instru_names(testqex)
        filt_names = qext.filt_names(testqex)
        wav = qext.wav(testqex)
        values = qext.values(testqex)
        sigma = qext.sigma(testqex)
        qext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
09e61d64   Ilyes Choubani   cleaned set_data ...
355
356
    ENDIF 
    ENDELSE
303de20d   Ilyes Choubani   updating check_data
357
    
09e61d64   Ilyes Choubani   cleaned set_data ...
358
359
    uexzone:
    ans=''
3ff9cac3   Ilyes Choubani   general update
360
361
    uext.values=data_ext.EXT_U
    uext.sigma=sqrt(abs(data_ext.sigextUU))
09e61d64   Ilyes Choubani   cleaned set_data ...
362
    ;Testig if there is EXT_P data. If not return empty value and end loop.
3ff9cac3   Ilyes Choubani   general update
363
    ind = where(data_ext.EXT_U NE la_undef() and finite(data_ext.EXT_U) EQ 1 and data_ext.sigextUU NE la_undef() and finite(data_ext.sigextUU) EQ 1,NU_ext) ;Testing if the data and errors are set
09e61d64   Ilyes Choubani   cleaned set_data ...
364
    IF NU_ext EQ 0 THEN BEGIN
3ff9cac3   Ilyes Choubani   general update
365
        uext = ptr_new()
303de20d   Ilyes Choubani   updating check_data
366
        goto, psiexzone
09e61d64   Ilyes Choubani   cleaned set_data ...
367
    ENDIF ELSE BEGIN
3ff9cac3   Ilyes Choubani   general update
368
    ind = where(finite(data_ext.EXT_U) EQ 0 or finite(data_ext.sigextUU) EQ 0, countnokuex) ;Format is not okay if there are nan values but the wrapper recognizes the la_undef() value 
09e61d64   Ilyes Choubani   cleaned set_data ...
369
    IF countnokuex NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
370
        message, "SED format isn't met. NaN value(s) detected in EXT_U data.",/continue
09e61d64   Ilyes Choubani   cleaned set_data ...
371
372
373
374
375
        message, 'Without automatic format fitering you cannot proceed.',/continue
        ans=''
        read, ans, prompt='Would you like automatic format filtering? (Y/N):' 
        IF strupcase(ans) EQ 'N' THEN BEGIN
            message,'You have chosen not to use automatic filtering. Please adjust the SED format.',/continue
266ae799   Ilyes Choubani   General update
376
            message, 'Aborting ...'
09e61d64   Ilyes Choubani   cleaned set_data ...
377
378
379
380
        ENDIF
    ENDIF
    IF strupcase(ans) EQ 'Y' THEN message, 'You have chosen automatic format filtering. Removal of undefined data points...',/continue $
    ELSE message, 'Data format is met. Removal of undefined data points...',/continue
3ff9cac3   Ilyes Choubani   general update
381
382
383
    tdaerruex=where((data_ext.sigextUU EQ la_undef() and data_ext.EXT_U NE la_undef()) or (data_ext.sigextUU NE la_undef() and data_ext.EXT_U EQ la_undef()),ctdaerruex)
    IF ctdaerruex NE 0 THEN message, 'Data and errors do not match. Undefinded data detected. Aborting...'
    testuex=where(data_ext.sigextUU NE 0. and finite(data_ext.sigextUU) EQ 1 and data_ext.sigextUU NE la_undef() and data_ext.EXT_U NE la_undef() and finite(data_ext.EXT_U) EQ 1,ctestuex)                
09e61d64   Ilyes Choubani   cleaned set_data ...
384
    IF ctestuex NE 0 THEN BEGIN 
3ff9cac3   Ilyes Choubani   general update
385
386
387
388
389
390
        instru_names = uext.instru_names(testuex)
        filt_names = uext.filt_names(testuex)
        wav = uext.wav(testuex)
        values = uext.values(testuex)
        sigma = uext.sigma(testuex)
        uext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
09e61d64   Ilyes Choubani   cleaned set_data ...
391
392
    ENDIF 
    ENDELSE
303de20d   Ilyes Choubani   updating check_data
393
394
395
    
    psiexzone: ;Starting the analogy here. 
    
04efbaa5   Ilyes Choubani   Correcting set_da...
396
    if NU_EXT eq 0 and NQ_EXT eq 0 then goto, end_ext
303de20d   Ilyes Choubani   updating check_data
397
398
399
400
401
402
    degtorad = !pi/180
    
    ;Generation of POLEXT and PSI_EXT data by default   
    ;Testing on the presence of covariances otherwise an error is returned
    ;The testing happens here because covariances are needed for the error estimation of EXT_P,PSI_EXT.   
    
0e955322   Ilyes Choubani   fixing small typo...
403
    tstcovuex = where((data_ext(testuex).sigextIU) EQ la_undef() or (data_ext(testuex).sigextQU) EQ la_undef(), ctcovuex)
303de20d   Ilyes Choubani   updating check_data
404
405
406
407
408
409
410
411
412
413
414
415
416
    if ctcovuex NE 0  then message, 'Covariance data is missing in the extinction data pertaining to the StokesU parameter. Data, errors and covariances do not match. Aborting...'
    tstcovqex = where((data_ext(testqex).sigextIQ) EQ la_undef() or (data_ext(testqex).sigextQU) EQ la_undef(), ctcovqex)
    if ctcovqex NE 0  then message, 'Covariance data is missing in the extinction pertaining to the StokesQ parameter. Data, errors and covariances do not match. Aborting...'
    
    ;Matching both filter and spectrum points 
    match2,uext.wav,qext.wav,qexpts,uexpts
    
    ;removing -1 values from the indices (qpts & upts)
    indnotqex = where(qexpts eq -1, cnotqex)
    if cnotqex ne 0 then remove, indnotqex, qexpts
    indnotuex = where(uexpts eq -1, cnotuex)
    if cnotuex ne 0 then remove, indnotuex, uexpts
    
0e955322   Ilyes Choubani   fixing small typo...
417
    match2, ext.wav, qext.wav(qexpts), qextmp, extpts
303de20d   Ilyes Choubani   updating check_data
418
419
420
421
422
423
424
    indnotx = where(extpts eq -1, cnotx)
    if cnotx ne 0 then remove, indnotx, extpts
    
    instru_names = uext.instru_names(uexpts)
    filt_names = uext.filt_names(uexpts)
    wav = uext.wav(uexpts)
    values = 0.5*atan(uext.values(uexpts),qext.values(qexpts))/degtorad
0e955322   Ilyes Choubani   fixing small typo...
425
    polar_variance_iqu2ippsi,ext.values(extpts),qext.values(qexpts),uext.values(uexpts),(data_ext(testex).sigextII)(extpts),(data_ext(testqex).sigextQQ)(qexpts),(data_ext(testuex).sigextUU)(uexpts),(data_ext(testex).sigextIQ)(extpts),(data_ext(testex).sigextIU)(extpts),(data_ext(testex).sigextQU)(extpts),variance_smallp,variance_psi,variance_largep
303de20d   Ilyes Choubani   updating check_data
426
    sigma = sqrt(variance_psi)
5a2643a4   Ilyes Choubani   cleaning/improvin...
427
    psi_ext={instru_names:instru_names,filt_names:filt_names,wav:wav,values:values,sigma:sigma}
07372e26   Ilyes Choubani   Corrected huge pl...
428
429


303de20d   Ilyes Choubani   updating check_data
430
    polext = psi_ext
0e955322   Ilyes Choubani   fixing small typo...
431
    polext.values = sqrt(uext.values(uexpts)^2+qext.values(qexpts)^2) 
303de20d   Ilyes Choubani   updating check_data
432
    polext.sigma = sqrt(variance_largep)
67bd858a   Ilyes Choubani   Replication of th...
433
434
435
436
437
    
    fpolext = psi_ext
    fpolext.values = polext.values/ext.values(extpts) 
    fpolext.sigma = sqrt(variance_smallp)
    
303de20d   Ilyes Choubani   updating check_data
438
439
    goto, end_ext
    
67bd858a   Ilyes Choubani   Replication of th...
440
    
09e61d64   Ilyes Choubani   cleaned set_data ...
441
442
    end_ext:
ENDIF
04efbaa5   Ilyes Choubani   Correcting set_da...
443
444
445

the_end:

09e61d64   Ilyes Choubani   cleaned set_data ...
446
447

END