LibOFX: ofx_preproc.cpp Source File

LibOFX

ofx_preproc.cpp
Go to the documentation of this file.
1 /***************************************************************************
2 ofx_preproc.cpp
3 -------------------
4 copyright : (C) 2002 by Benoit Gr�oir
5 email : benoitg@coeus.ca
6 ***************************************************************************/
12 /***************************************************************************
13 * *
14 * This program is free software; you can redistribute it and/or modify *
15 * it under the terms of the GNU General Public License as published by *
16 * the Free Software Foundation; either version 2 of the License, or *
17 * (at your option) any later version. *
18 * *
19 ***************************************************************************/
20 #include "../config.h"
21 #include <iostream>
22 #include <fstream>
23 #include <cstdlib>
24 #include <stdio.h>
25 #include <sstream>
26 #include <string>
27 #include "ParserEventGeneratorKit.h"
28 #include "libofx.h"
29 #include "messages.hh"
30 #include "ofx_sgml.hh"
31 #include "ofc_sgml.hh"
32 #include "ofx_preproc.hh"
33 #include "ofx_utilities.hh"
34 #ifdef HAVE_ICONV
35 #include <iconv.h>
36 #endif
37
38 #ifdef __WIN32__
39 # define DIRSEP "\\"
40 #else
41 # define DIRSEP "/"
42 #endif
43
44 #ifdef __WIN32__
45 # include "win32.hh"
46 # include <windows.h> // for GetModuleFileName()
47 # undef ERROR
48 # undef DELETE
49 #endif
50
51 #define LIBOFX_DEFAULT_INPUT_ENCODING "CP1252"
52 #define LIBOFX_DEFAULT_OUTPUT_ENCODING "UTF-8"
53
54 using namespace std;
58 #ifdef MAKEFILE_DTD_PATH
59 const int DTD_SEARCH_PATH_NUM = 4;
60 #else
61 const int DTD_SEARCH_PATH_NUM = 3;
62 #endif
63
67 const char *DTD_SEARCH_PATH[DTD_SEARCH_PATH_NUM] =
68 {
69 #ifdef MAKEFILE_DTD_PATH
70 MAKEFILE_DTD_PATH ,
71 #endif
72 "/usr/local/share/libofx/dtd",
73 "/usr/share/libofx/dtd",
74 "~"
75 };
76
81 int ofx_proc_file(LibofxContextPtr ctx, const char * p_filename)
82 {
83 LibofxContext *libofx_context;
84 bool ofx_start = false;
85 bool ofx_end = false;
86 bool file_is_xml = false;
87
88 ifstream input_file;
89 ofstream tmp_file;
90 char *filenames[3];
91 char tmp_filename[256];
92 int tmp_file_fd;
93 #ifdef HAVE_ICONV
94 iconv_t conversion_descriptor;
95 #endif
96 libofx_context = (LibofxContext*)ctx;
97
98 if (p_filename != NULL && strcmp(p_filename, "") != 0)
99 {
100 message_out(DEBUG, string("ofx_proc_file():Opening file: ") + p_filename);
101
102 input_file.open(p_filename);
103 if (!input_file)
104 {
105 message_out(ERROR, "ofx_proc_file():Unable to open the input file " + string(p_filename));
106 }
107
108 mkTempFileName("libofxtmpXXXXXX", tmp_filename, sizeof(tmp_filename));
109
110 message_out(DEBUG, "ofx_proc_file(): Creating temp file: " + string(tmp_filename));
111 #ifdef __WIN32__
112 tmp_file_fd = mkstemp_win32(tmp_filename);
113 #else
114 tmp_file_fd = mkstemp(tmp_filename);
115 #endif
116 if (tmp_file_fd)
117 {
118 tmp_file.open(tmp_filename);
119 if (!tmp_file)
120 {
121 message_out(ERROR, "ofx_proc_file():Unable to open the created temp file " + string(tmp_filename));
122 return -1;
123 }
124 }
125 else
126 {
127 message_out(ERROR, "ofx_proc_file():Unable to create a temp file at " + string(tmp_filename));
128 return -1;
129 }
130
131 if (input_file && tmp_file)
132 {
133 int header_separator_idx;
134 string header_name;
135 string header_value;
136 string ofx_encoding;
137 string ofx_charset;
138 do
139 {
140 stringbuf buffer;
141 string s_buffer;
142 input_file.get(buffer, '\n');
143 //cout<< "got: \"" << buffer<<"\"\n";
144 s_buffer = buffer.str();
145
146 // Watch out: If input_file is in eof(), any subsequent read or
147 // peek() will fail and we must exit this loop.
148 if (!input_file.eof())
149 {
150 //cout<<"input_file.gcount(): "<<input_file.gcount()<< " s_buffer.size=" << s_buffer.size()<<" sizeof(buffer): "<<sizeof(buffer) << " peek=\"" << int(input_file.peek()) << "\"" <<endl;
151 if (input_file.fail()) // If no characters were extracted above, the failbit is set.
152 {
153 // No characters extracted means that we've reached the newline
154 // delimiter (because we already checked for EOF). We will check
155 // for and remove that newline in the next if-clause, but must
156 // remove the failbit so that peek() will work again.
157 input_file.clear();
158 }
159
160 // Is the next character really the newline?
161 if (input_file.peek() == '\n')
162 {
163 // Yes. Then discard that newline character from the stream
164 input_file.get();
165 }
166 }
167
168 if (ofx_start == false && (s_buffer.find("<?xml") != string::npos))
169 {
170 message_out(DEBUG, "ofx_proc_file(): File is an actual XML file, iconv conversion will be skipped.");
171 file_is_xml = true;
172 }
173
174 int ofx_start_idx;
175 if (ofx_start == false)
176 {
177 if (
178 (libofx_context->currentFileType() == OFX &&
179 ((ofx_start_idx = s_buffer.find("<OFX>")) != string::npos ||
180 (ofx_start_idx = s_buffer.find("<ofx>")) != string::npos))
181 ||
182 (libofx_context->currentFileType() == OFC &&
183 ((ofx_start_idx = s_buffer.find("<OFC>")) != string::npos ||
184 (ofx_start_idx = s_buffer.find("<ofc>")) != string::npos))
185 )
186 {
187 ofx_start = true;
188 if (file_is_xml == false)
189 {
190 s_buffer.erase(0, ofx_start_idx); //Fix for really broken files that don't have a newline after the header.
191 }
192 message_out(DEBUG, "ofx_proc_file():<OFX> or <OFC> has been found");
193
194 if (file_is_xml == true)
195 {
196 static char sp_charset_fixed[] = "SP_CHARSET_FIXED=1";
197 if (putenv(sp_charset_fixed) != 0)
198 {
199 message_out(ERROR, "ofx_proc_file(): putenv failed");
200 }
201 /* Normally the following would be "xml".
202 * Unfortunately, opensp's generic api will garble UTF-8 if this is
203 * set to xml. So we set any single byte encoding to avoid messing
204 * up UTF-8. Unfortunately this means that non-UTF-8 files will not
205 * get properly translated. We'd need to manually detect the
206 * encoding in the XML header and convert the xml with iconv like we
207 * do for SGML to work around the problem. Most unfortunate. */
208 static char sp_encoding[] = "SP_ENCODING=ms-dos";
209 if (putenv(sp_encoding) != 0)
210 {
211 message_out(ERROR, "ofx_proc_file(): putenv failed");
212 }
213 }
214 else
215 {
216 static char sp_charset_fixed[] = "SP_CHARSET_FIXED=1";
217 if (putenv(sp_charset_fixed) != 0)
218 {
219 message_out(ERROR, "ofx_proc_file(): putenv failed");
220 }
221 static char sp_encoding[] = "SP_ENCODING=ms-dos"; //Any single byte encoding will do, we don't want opensp messing up UTF-8;
222 if (putenv(sp_encoding) != 0)
223 {
224 message_out(ERROR, "ofx_proc_file(): putenv failed");
225 }
226 #ifdef HAVE_ICONV
227 string fromcode;
228 string tocode;
229 if (ofx_encoding.compare("USASCII") == 0)
230 {
231 if (ofx_charset.compare("ISO-8859-1") == 0 || ofx_charset.compare("8859-1") == 0)
232 {
233 //Only "ISO-8859-1" is actually a legal value, but since the banks follows the spec SO well...
234 fromcode = "ISO-8859-1";
235 }
236 else if (ofx_charset.compare("1252") == 0 || ofx_charset.compare("CP1252") == 0)
237 {
238 //Only "1252" is actually a legal value, but since the banks follows the spec SO well...
239 fromcode = "CP1252";
240 }
241 else if (ofx_charset.compare("NONE") == 0)
242 {
243 fromcode = LIBOFX_DEFAULT_INPUT_ENCODING;
244 }
245 else
246 {
247 fromcode = LIBOFX_DEFAULT_INPUT_ENCODING;
248 }
249 }
250 else if (ofx_encoding.compare("UTF-8") == 0 || ofx_encoding.compare("UNICODE") == 0)
251 {
252 //While "UNICODE" isn't a legal value, some cyrilic files do specify it as such...
253 fromcode = "UTF-8";
254 }
255 else
256 {
257 fromcode = LIBOFX_DEFAULT_INPUT_ENCODING;
258 }
259 tocode = LIBOFX_DEFAULT_OUTPUT_ENCODING;
260 message_out(DEBUG, "ofx_proc_file(): Setting up iconv for fromcode: " + fromcode + ", tocode: " + tocode);
261 conversion_descriptor = iconv_open (tocode.c_str(), fromcode.c_str());
262 #endif
263 }
264 }
265 else
266 {
267 //We are still in the headers
268 if ((header_separator_idx = s_buffer.find(':')) != string::npos)
269 {
270 //Header processing
271 header_name.assign(s_buffer.substr(0, header_separator_idx));
272 header_value.assign(s_buffer.substr(header_separator_idx + 1));
273 while ( header_value[header_value.length() -1 ] == '\n' ||
274 header_value[header_value.length() -1 ] == '\r' )
275 header_value.erase(header_value.length() - 1);
276 message_out(DEBUG, "ofx_proc_file():Header: " + header_name + " with value: " + header_value + " has been found");
277 if (header_name.compare("ENCODING") == 0)
278 {
279 ofx_encoding.assign(header_value);
280 }
281 if (header_name.compare("CHARSET") == 0)
282 {
283 ofx_charset.assign(header_value);
284 }
285 }
286 }
287 }
288
289 if (file_is_xml == true || (ofx_start == true && ofx_end == false))
290 {
291 if (ofx_start == true)
292 {
293 /* The above test won't help us if the <OFX> tag is on the same line
294 * as the xml header, but as opensp can't be used to parse it anyway
295 * this isn't a great loss for now.
296 */
297 s_buffer = sanitize_proprietary_tags(s_buffer);
298 if (s_buffer.empty())
299 continue;
300 }
301 //cout<< s_buffer<<"\n";
302 if (file_is_xml == false)
303 {
304 #ifdef HAVE_ICONV
305 size_t inbytesleft = s_buffer.size();
306 size_t outbytesleft = inbytesleft * 2 - 1;
307 char * iconv_buffer = (char*) malloc (inbytesleft * 2);
308 memset(iconv_buffer, 0, inbytesleft * 2);
309 #if defined(__sun) || defined(__NetBSD__)
310 const char * inchar = (const char *)s_buffer.c_str();
311 #else
312 char * inchar = (char *)s_buffer.c_str();
313 #endif
314 char * outchar = iconv_buffer;
315 int iconv_retval = iconv (conversion_descriptor,
316 &inchar, &inbytesleft,
317 &outchar, &outbytesleft);
318 if (iconv_retval == -1)
319 {
320 message_out(ERROR, "ofx_proc_file(): Iconv conversion error");
321 }
322 // All validly converted bytes will be copied to the
323 // original buffer
324 s_buffer = std::string(iconv_buffer, outchar - iconv_buffer);
325 free (iconv_buffer);
326 #endif
327 }
328 //cout << s_buffer << "\n";
329 tmp_file << s_buffer << endl;
330 }
331
332 if (ofx_start == true &&
333 (
334 (libofx_context->currentFileType() == OFX &&
335 ((ofx_start_idx = s_buffer.find("</OFX>")) != string::npos ||
336 (ofx_start_idx = s_buffer.find("</ofx>")) != string::npos))
337 || (libofx_context->currentFileType() == OFC &&
338 ((ofx_start_idx = s_buffer.find("</OFC>")) != string::npos ||
339 (ofx_start_idx = s_buffer.find("</ofc>")) != string::npos))
340 )
341 )
342 {
343 ofx_end = true;
344 message_out(DEBUG, "ofx_proc_file():</OFX> or </OFC> has been found");
345 }
346
347 }
348 while (!input_file.eof() && !input_file.bad());
349 }
350 input_file.close();
351 tmp_file.close();
352 #ifdef HAVE_ICONV
353 if (file_is_xml == false)
354 {
355 iconv_close(conversion_descriptor);
356 }
357 #endif
358 char filename_openspdtd[255];
359 char filename_dtd[255];
360 char filename_ofx[255];
361 strncpy(filename_openspdtd, find_dtd(ctx, OPENSPDCL_FILENAME).c_str(), 255); //The opensp sgml dtd file
362 if (libofx_context->currentFileType() == OFX)
363 {
364 strncpy(filename_dtd, find_dtd(ctx, OFX160DTD_FILENAME).c_str(), 255); //The ofx dtd file
365 }
366 else if (libofx_context->currentFileType() == OFC)
367 {
368 strncpy(filename_dtd, find_dtd(ctx, OFCDTD_FILENAME).c_str(), 255); //The ofc dtd file
369 }
370 else
371 {
372 message_out(ERROR, string("ofx_proc_file(): Error unknown file format for the OFX parser"));
373 }
374
375 if ((string)filename_dtd != "" && (string)filename_openspdtd != "")
376 {
377 strncpy(filename_ofx, tmp_filename, 255); //The processed ofx file
378 filenames[0] = filename_openspdtd;
379 filenames[1] = filename_dtd;
380 filenames[2] = filename_ofx;
381 if (libofx_context->currentFileType() == OFX)
382 {
383 ofx_proc_sgml(libofx_context, 3, filenames);
384 }
385 else if (libofx_context->currentFileType() == OFC)
386 {
387 ofc_proc_sgml(libofx_context, 3, filenames);
388 }
389 else
390 {
391 message_out(ERROR, string("ofx_proc_file(): Error unknown file format for the OFX parser"));
392 }
393 if (remove(tmp_filename) != 0)
394 {
395 message_out(ERROR, "ofx_proc_file(): Error deleting temporary file " + string(tmp_filename));
396 }
397 }
398 else
399 {
400 message_out(ERROR, "ofx_proc_file(): FATAL: Missing DTD, aborting");
401 }
402 }
403 else
404 {
405 message_out(ERROR, "ofx_proc_file():No input file specified");
406 }
407 return 0;
408 }
409
410 /* Searches input string for an opening or closing tag starting from pos_start.
411 * If found will return the tag_name and pos_start will be set to the string
412 * of the starting <, pos_end to the position after the closing '>'
413 * If the tag doesn't have a closing '>', pos_end will be set to string::npos.
414 */
415 static string find_tag_open (string& input_string, size_t& pos_start, size_t& pos_end)
416 {
417 pos_start = input_string.find ('<', pos_start);
418
419 if (pos_start == string::npos)
420 {
421 pos_end = string::npos;
422 return string();
423 }
424
425 pos_end = input_string.find ('>', pos_start + 1);
426 if (pos_end != string::npos)
427 pos_end = pos_end + 1;
428 size_t tag_size = (pos_end - 1) - (pos_start + 1);
429 return input_string.substr(pos_start + 1 , tag_size);
430 }
431
432 /* Searches input string for a closing tag matching tag_name starting at pos.
433 * If found pos will be set to the position right after of the closing '>'
434 * If no matching closing tag is found pos will be set to the start of the next
435 * opening or closing tag found.
436 */
437 static void find_tag_close (string& input_string, string& tag_name, size_t& pos)
438 {
439 size_t start_idx = input_string.find ("</" + tag_name + ">", pos);
440
441 if (start_idx == string::npos)
442 {
443 start_idx = pos;
444 size_t end_idx;
445 string new_tag_name = find_tag_open (input_string, start_idx, end_idx);
446 if (!new_tag_name.empty())
447 {
448 message_out(DEBUG, "find_tag_close() fell back to next open tag: " + new_tag_name);
449 // find_tag_open returns the *end* of an opening tag, but in this
450 // case we want its start, so we need to rewind a bit..
451 pos = start_idx;
452 //printf("find_tag_close() returning pos after fallback: %d\n",pos);
453 }
454 else
455 {
456 pos = input_string.length();
457 }
458 }
459 else
460 {
461 pos = start_idx + tag_name.length() + 3;
462 }
463 return;
464 }
465
466
478 string sanitize_proprietary_tags(string input_string)
479 {
480 size_t last_known_good_pos = 0;
481 size_t open_tag_start_pos = last_known_good_pos;
482 size_t open_tag_end_pos;
483 size_t close_tag_end_pos;
484
485 string tag_name = find_tag_open(input_string, open_tag_start_pos, open_tag_end_pos);
486 while (!tag_name.empty())
487 {
488 // Determine whether the current tag is proprietary.
489 if ((tag_name.find('.') != string::npos) || // tag has a . in the name
490 (tag_name == "CATEGORY")) // Chase bank started setting these in 2017
491 {
492 close_tag_end_pos = open_tag_end_pos;
493 find_tag_close (input_string, tag_name, close_tag_end_pos);
494 size_t tag_size = close_tag_end_pos - open_tag_start_pos;
495 string prop_tag = input_string.substr(open_tag_start_pos, tag_size);
496 message_out(INFO, "sanitize_proprietary_tags() removed: " + prop_tag);
497 input_string.erase(open_tag_start_pos, tag_size);
498 last_known_good_pos = open_tag_start_pos;
499 }
500 else
501 {
502 last_known_good_pos = open_tag_end_pos;
503 }
504 tag_name.clear();
505 open_tag_start_pos = last_known_good_pos;
506 if (last_known_good_pos != string::npos)
507 tag_name = find_tag_open(input_string, open_tag_start_pos, open_tag_end_pos);
508 }
509 return input_string;
510 }
511
512
513 #ifdef __WIN32__
514 static std::string get_dtd_installation_directory()
515 {
516 // Partial implementation of
517 // http://developer.gnome.org/doc/API/2.0/glib/glib-Windows-Compatibility-Functions.html#g-win32-get-package-installation-directory
518 char ch_fn[MAX_PATH], *p;
519 std::string str_fn;
520
521 if (!GetModuleFileName(NULL, ch_fn, MAX_PATH)) return "";
522
523 if ((p = strrchr(ch_fn, '\\')) != NULL)
524 * p = '\0';
525
526 p = strrchr(ch_fn, '\\');
527 if (p && (_stricmp(p + 1, "bin") == 0 ||
528 _stricmp(p + 1, "lib") == 0))
529 *p = '\0';
530
531 str_fn = ch_fn;
532 str_fn += "\\share\\libofx\\dtd";
533
534 return str_fn;
535 }
536 #endif
537
538
551 std::string find_dtd(LibofxContextPtr ctx, const std::string& dtd_filename)
552 {
553 string dtd_path_filename;
554 char *env_dtd_path;
555
556 dtd_path_filename = reinterpret_cast<const LibofxContext*>(ctx)->dtdDir();
557 if (!dtd_path_filename.empty())
558 {
559 dtd_path_filename.append(dtd_filename);
560 ifstream dtd_file(dtd_path_filename.c_str());
561 if (dtd_file)
562 {
563 message_out(STATUS, "find_dtd():DTD found: " + dtd_path_filename);
564 return dtd_path_filename;
565 }
566 }
567
568 #ifdef __WIN32__
569 dtd_path_filename = get_dtd_installation_directory();
570 if (!dtd_path_filename.empty())
571 {
572 dtd_path_filename.append(DIRSEP);
573 dtd_path_filename.append(dtd_filename);
574 ifstream dtd_file(dtd_path_filename.c_str());
575 if (dtd_file)
576 {
577 message_out(STATUS, "find_dtd():DTD found: " + dtd_path_filename);
578 return dtd_path_filename;
579 }
580 }
581 #endif
582 /* Search in environement variable OFX_DTD_PATH */
583 env_dtd_path = getenv("OFX_DTD_PATH");
584 if (env_dtd_path)
585 {
586 dtd_path_filename.append(env_dtd_path);
587 dtd_path_filename.append(DIRSEP);
588 dtd_path_filename.append(dtd_filename);
589 ifstream dtd_file(dtd_path_filename.c_str());
590 if (!dtd_file)
591 {
592 message_out(STATUS, "find_dtd():OFX_DTD_PATH env variable was was present, but unable to open the file " + dtd_path_filename);
593 }
594 else
595 {
596 message_out(STATUS, "find_dtd():DTD found: " + dtd_path_filename);
597 return dtd_path_filename;
598 }
599 }
600
601 for (int i = 0; i < DTD_SEARCH_PATH_NUM; i++)
602 {
603 dtd_path_filename = DTD_SEARCH_PATH[i];
604 dtd_path_filename.append(DIRSEP);
605 dtd_path_filename.append(dtd_filename);
606 ifstream dtd_file(dtd_path_filename.c_str());
607 if (!dtd_file)
608 {
609 message_out(DEBUG, "find_dtd():Unable to open the file " + dtd_path_filename);
610 }
611 else
612 {
613 message_out(STATUS, "find_dtd():DTD found: " + dtd_path_filename);
614 return dtd_path_filename;
615 }
616 }
617
618 /* Last resort, look in source tree relative path (useful for development) */
619 dtd_path_filename = "";
620 dtd_path_filename.append("..");
621 dtd_path_filename.append(DIRSEP);
622 dtd_path_filename.append("dtd");
623 dtd_path_filename.append(DIRSEP);
624 dtd_path_filename.append(dtd_filename);
625 ifstream dtd_file(dtd_path_filename.c_str());
626 if (!dtd_file)
627 {
628 message_out(DEBUG, "find_dtd(): Unable to open the file " + dtd_path_filename + ", most likely we are not in the source tree.");
629 }
630 else
631 {
632 message_out(STATUS, "find_dtd():DTD found: " + dtd_path_filename);
633 return dtd_path_filename;
634 }
635
636
637 message_out(ERROR, "find_dtd():Unable to find the DTD named " + dtd_filename);
638 return "";
639 }
INFO
Definition: messages.hh:32
ofx_proc_file
int ofx_proc_file(LibofxContextPtr ctx, const char *p_filename)
File pre-processing of OFX AND for OFC files.
Definition: ofx_preproc.cpp:81
std
STATUS
Definition: messages.hh:31
OFX
Definition: inc/libofx.h:126
LibofxContext
Definition: context.hh:25
DTD_SEARCH_PATH_NUM
const int DTD_SEARCH_PATH_NUM
The number of different paths to search for DTDs.
Definition: ofx_preproc.cpp:61
message_out
int message_out(OfxMsgType error_type, const string message)
Message output function.
Definition: messages.cpp:61
DEBUG
Definition: messages.hh:25
ofx_sgml.hh
OFX/SGML parsing functionnality.
DTD_SEARCH_PATH
const char * DTD_SEARCH_PATH[DTD_SEARCH_PATH_NUM]
The list of paths to search for the DTDs.
Definition: ofx_preproc.cpp:67
ofc_proc_sgml
int ofc_proc_sgml(LibofxContext *libofx_context, int argc, char *const *argv)
Parses a DTD and OFX file(s)
Definition: ofc_sgml.cpp:353
ofx_utilities.hh
Various simple functions for type conversion & al.
ofx_proc_sgml
int ofx_proc_sgml(LibofxContext *libofx_context, int argc, char *const *argv)
Parses a DTD and OFX file(s)
Definition: ofx_sgml.cpp:372
ERROR
Definition: messages.hh:34
OFC
Definition: inc/libofx.h:127
sanitize_proprietary_tags
string sanitize_proprietary_tags(string input_string)
Removes proprietary tags and comments.
Definition: ofx_preproc.cpp:478
ofc_sgml.hh
OFX/SGML parsing functionnality.
messages.hh
Message IO functionality.
ofx_preproc.hh
Preprocessing of the OFX files before parsing.
find_dtd
std::string find_dtd(LibofxContextPtr ctx, const std::string &dtd_filename)
Find the appropriate DTD for the file version.
Definition: ofx_preproc.cpp:551