CDPP / TREPS

Commit 795653bf9d4e6411f471b20ceb079eeaef923a44

Authored by Laurent BEIGBEDER
1 parent c8c2b610
Exists in master

9763 - checking wget output result

Showing 1 changed file with 204 additions and 194 deletions   Show diff stats
Inline Side-by-side
server/kernel/src/DataRetriever/DataRetriever.cpp
  Diff comments   View file @795653b
1   -#include "DataRetriever.h"
2   -
3   -#include <fstream>
4   -#include <cstdlib>
5   -
6   -#include "../Common/Toolbox.h"
7   -
8   -using namespace std;
9   -using namespace TREPS::Common;
10   -
11   -namespace TREPS
12   -{
13   - namespace DataRetriever
14   - {
15   - DataRetrieverClass::DataRetrieverClass(void): app(NULL), errorMsg("")
16   - {
17   - this->app = ApplicationClass::getInstance();
18   - }
19   -
20   - bool DataRetrieverClass::copyLocalFile(const char *file_path, const char *dest_dir, const char *dest_name, bool checkSize)
21   - {
22   - this->errorMsg = "";
23   -
24   - if (checkSize)
25   - {
26   - //test the file size
27   - int fileSize = getFileSize(file_path);
28   -
29   - if (fileSize < 0)
30   - {
31   - this->errorMsg = "Cannot get file size";
32   - LOG4CXX_ERROR(this->app->getLog()->getPtr(), this->errorMsg);
33   - return false;
34   - }
35   -
36   - if (fileSize > this->app->getConf()->getMaxSourceFileSize())
37   - {
38   - this->errorMsg = "File too large (limitation to ";
39   - this->errorMsg += intToStr(this->app->getConf()->getMaxSourceFileSize());
40   - this->errorMsg += " octets)";
41   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
42   - return false;
43   - }
44   - }
45   -
46   - LOG4CXX_INFO(this->app->getLog()->getPtr(),"Try to copy local file " << file_path << " to dir " << dest_dir);
47   -
48   - //copy the file
49   - ifstream source(file_path, ios::binary);
50   -
51   - if (!source.good())
52   - {
53   - this->errorMsg = "Error to load source file";
54   - LOG4CXX_ERROR(this->app->getLog()->getPtr(), this->errorMsg);
55   - return false;
56   - }
57   -
58   - string destPath = getPathCorrection(dest_dir);
59   -
60   - destPath += dest_name;
61   -
62   - ofstream dest(destPath.c_str(), ios::binary);
63   -
64   - dest << source.rdbuf();
65   -
66   - source.close();
67   - dest.close();
68   -
69   - return true;
70   - }
71   -
72   - bool DataRetrieverClass::copyWebFile(const char *file_url, const char *dest_dir, const char *dest_name, bool checkSize)
73   - {
74   - this->errorMsg = "";
75   -
76   - //use the call of wget system command to copy a web file
77   - string cmd = "wget ";
78   -
79   - string proxyUser = this->app->getConf()->getProxyUsername();
80   - string proxyPwd = this->app->getConf()->getProxyPassword();
81   -
82   - if (proxyUser.compare("") != 0)
83   - {
84   - cmd += "--proxy-user=";
85   - cmd += proxyUser;
86   - cmd += " ";
87   - }
88   -
89   - if (proxyPwd.compare("") != 0)
90   - {
91   - cmd += "--proxy-password=";
92   - cmd += proxyPwd;
93   - cmd += " ";
94   - }
95   -
96   - if (checkSize)
97   - {
98   - //test file size
99   - string cmd_size = cmd;
100   - cmd_size += "--spider \"";
101   - cmd_size += file_url;
102   -
103   - //for english server
104   - string cmd_size_english = cmd_size;
105   - cmd_size_english += "\" 2>&1 | grep Length | awk '{print $2}'";
106   -
107   - //for french server
108   - string cmd_size_french = cmd_size;
109   - cmd_size_french += "\" 2>&1 | grep Longueur | awk '{print $2}'";
110   -
111   - string output;
112   - //run command for french server
113   - int status = executeSystemCommand(cmd_size_french.c_str(), output);
114   -
115   - if (status != 0)
116   - {
117   - this->errorMsg = "Cannot retrieve web file";
118   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
119   - return false;
120   - }
121   -
122   - if (output.compare("") == 0)
123   - {
124   - //if no output, try to run english command
125   - status = executeSystemCommand(cmd_size_english.c_str(), output);
126   -
127   - if (status != 0)
128   - {
129   - this->errorMsg = "Cannot retrieve web file";
130   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
131   - return false;
132   - }
133   -
134   - if (output.compare("") == 0)
135   - {
136   - this->errorMsg = "Cannot retrieve web file";
137   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
138   - return false;
139   - }
140   - }
141   -
142   - long int fileSize = atol(output.c_str());
143   -
144   - if (fileSize <= 0)
145   - {
146   - this->errorMsg = "Internal error to get the file size.";
147   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
148   - return false;
149   - }
150   -
151   - if (fileSize > this->app->getConf()->getMaxSourceFileSize())
152   - {
153   - this->errorMsg = "File too large (limitation to ";
154   - this->errorMsg += intToStr(this->app->getConf()->getMaxSourceFileSize());
155   - this->errorMsg += " octets)";
156   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
157   - return false;
158   - }
159   - }
160   -
161   - //copy the file
162   - string destPath = getPathCorrection(dest_dir);
163   -
164   - destPath += dest_name;
165   -
166   - cmd += "-O ";
167   - cmd += destPath;
168   -
169   - cmd += " \"";
170   - cmd += file_url;
171   - cmd += "\"";
172   -
173   - string output;
174   - int status = executeSystemCommand(cmd.c_str(),output);
175   -
176   - if (status != 0)
177   - {
178   - this->errorMsg = "Cannot copy web file ";
179   - this->errorMsg += file_url;
180   - this->errorMsg += " - System command error : ";
181   - this->errorMsg += intToStr(status);
182   - LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
183   - return false;
184   - }
185   -
186   - return true;
187   - }
188   -
189   - string DataRetrieverClass::getErrorMsg(void)
190   - {
191   - return this->errorMsg;
192   - }
193   - }
194   -}
  1 +#include "DataRetriever.h"
  2 +
  3 +#include <fstream>
  4 +#include <cstdlib>
  5 +
  6 +#include "../Common/Toolbox.h"
  7 +
  8 +using namespace std;
  9 +using namespace TREPS::Common;
  10 +
  11 +namespace TREPS
  12 +{
  13 + namespace DataRetriever
  14 + {
  15 + DataRetrieverClass::DataRetrieverClass(void): app(NULL), errorMsg("")
  16 + {
  17 + this->app = ApplicationClass::getInstance();
  18 + }
  19 +
  20 + bool DataRetrieverClass::copyLocalFile(const char *file_path, const char *dest_dir, const char *dest_name, bool checkSize)
  21 + {
  22 + this->errorMsg = "";
  23 +
  24 + if (checkSize)
  25 + {
  26 + //test the file size
  27 + int fileSize = getFileSize(file_path);
  28 +
  29 + if (fileSize < 0)
  30 + {
  31 + this->errorMsg = "Cannot get file size";
  32 + LOG4CXX_ERROR(this->app->getLog()->getPtr(), this->errorMsg);
  33 + return false;
  34 + }
  35 +
  36 + if (fileSize > this->app->getConf()->getMaxSourceFileSize())
  37 + {
  38 + this->errorMsg = "File too large (limitation to ";
  39 + this->errorMsg += intToStr(this->app->getConf()->getMaxSourceFileSize());
  40 + this->errorMsg += " octets)";
  41 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  42 + return false;
  43 + }
  44 + }
  45 +
  46 + LOG4CXX_INFO(this->app->getLog()->getPtr(),"Try to copy local file " << file_path << " to dir " << dest_dir);
  47 +
  48 + //copy the file
  49 + ifstream source(file_path, ios::binary);
  50 +
  51 + if (!source.good())
  52 + {
  53 + this->errorMsg = "Error to load source file";
  54 + LOG4CXX_ERROR(this->app->getLog()->getPtr(), this->errorMsg);
  55 + return false;
  56 + }
  57 +
  58 + string destPath = getPathCorrection(dest_dir);
  59 +
  60 + destPath += dest_name;
  61 +
  62 + ofstream dest(destPath.c_str(), ios::binary);
  63 +
  64 + dest << source.rdbuf();
  65 +
  66 + source.close();
  67 + dest.close();
  68 +
  69 + return true;
  70 + }
  71 +
  72 + bool DataRetrieverClass::copyWebFile(const char *file_url, const char *dest_dir, const char *dest_name, bool checkSize)
  73 + {
  74 + this->errorMsg = "";
  75 +
  76 + //use the call of wget system command to copy a web file
  77 + string cmd = "wget ";
  78 +
  79 + string proxyUser = this->app->getConf()->getProxyUsername();
  80 + string proxyPwd = this->app->getConf()->getProxyPassword();
  81 +
  82 + if (proxyUser.compare("") != 0)
  83 + {
  84 + cmd += "--proxy-user=";
  85 + cmd += proxyUser;
  86 + cmd += " ";
  87 + }
  88 +
  89 + if (proxyPwd.compare("") != 0)
  90 + {
  91 + cmd += "--proxy-password=";
  92 + cmd += proxyPwd;
  93 + cmd += " ";
  94 + }
  95 +
  96 + if (checkSize)
  97 + {
  98 + //test file size
  99 + string cmd_size = cmd;
  100 + cmd_size += "--spider \"";
  101 + cmd_size += file_url;
  102 +
  103 + //for english server
  104 + string cmd_size_english = cmd_size;
  105 + cmd_size_english += "\" 2>&1 | grep Length | awk '{print $2}'";
  106 +
  107 + //for french server
  108 + string cmd_size_french = cmd_size;
  109 + cmd_size_french += "\" 2>&1 | grep Longueur | awk '{print $2}'";
  110 +
  111 + string output;
  112 + //run command for french server
  113 + int status = executeSystemCommand(cmd_size_french.c_str(), output);
  114 +
  115 + if (status != 0)
  116 + {
  117 + this->errorMsg = "Cannot retrieve web file";
  118 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  119 + return false;
  120 + }
  121 +
  122 + if (output.compare("") == 0)
  123 + {
  124 + //if no output, try to run english command
  125 + status = executeSystemCommand(cmd_size_english.c_str(), output);
  126 +
  127 + if (status != 0)
  128 + {
  129 + this->errorMsg = "Cannot retrieve web file";
  130 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  131 + return false;
  132 + }
  133 +
  134 + if (output.compare("") == 0)
  135 + {
  136 + this->errorMsg = "Cannot retrieve web file";
  137 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  138 + return false;
  139 + }
  140 + }
  141 +//9763 - checking wget output result
  142 +// In the case of AMDA HAPI, wget does not return the length of the data, Length: unspecified
  143 +// if Length is unspecified, then no checking file size
  144 + if( output.find("unspecified") != 0){
  145 + long int fileSize = atol(output.c_str());
  146 +
  147 + if (fileSize <= 0)
  148 + {
  149 + this->errorMsg = "Internal error to get the file size.";
  150 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  151 + return false;
  152 + }
  153 +
  154 + if (fileSize > this->app->getConf()->getMaxSourceFileSize())
  155 + {
  156 + this->errorMsg = "File too large (limitation to ";
  157 + this->errorMsg += intToStr(this->app->getConf()->getMaxSourceFileSize());
  158 + this->errorMsg += " octets)";
  159 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  160 + return false;
  161 + }
  162 +
  163 + }else{
  164 + LOG4CXX_INFO(this->app->getLog()->getPtr()," copyWebFile Length is unspecified: no file size checking" );
  165 + }
  166 +
  167 + }
  168 +
  169 +
  170 +
  171 + //copy the file
  172 + string destPath = getPathCorrection(dest_dir);
  173 +
  174 + destPath += dest_name;
  175 +
  176 + cmd += "-O ";
  177 + cmd += destPath;
  178 +
  179 + cmd += " \"";
  180 + cmd += file_url;
  181 + cmd += "\"";
  182 +
  183 + string output;
  184 + int status = executeSystemCommand(cmd.c_str(),output);
  185 +
  186 + if (status != 0)
  187 + {
  188 + this->errorMsg = "Cannot copy web file ";
  189 + this->errorMsg += file_url;
  190 + this->errorMsg += " - System command error : ";
  191 + this->errorMsg += intToStr(status);
  192 + LOG4CXX_ERROR(this->app->getLog()->getPtr(),this->errorMsg);
  193 + return false;
  194 + }
  195 +
  196 + return true;
  197 + }
  198 +
  199 + string DataRetrieverClass::getErrorMsg(void)
  200 + {
  201 + return this->errorMsg;
  202 + }
  203 + }
  204 +}
... ...