t_parse_csv.cpp

Go to the documentation of this file.

/*****************************************************************************\
*                                                                             *
*  Name   : test parsing of csv                                               *
*  Author : Chris Koeritz                                                     *
*                                                                             *
*  Purpose:                                                                   *
*                                                                             *
*    Checks that the CSV parsing function handles a few common scenarios.     *
*                                                                             *
*******************************************************************************
* Copyright (c) 2005-$now By Author.  This program is free software; you can  *
* redistribute it and/or modify it under the terms of the GNU General Public  *
* License as published by the Free Software Foundation; either version 2 of   *
* the License or (at your option) any later version.  This is online at:      *
*     http://www.fsf.org/copyleft/gpl.html                                    *
\*****************************************************************************/

#include <basis/function.h>
#include <basis/guards.h>
#include <basis/istring.h>
#include <basis/string_array.h>
#include <data_struct/static_memory_gremlin.h>
#include <loggers/console_logger.h>
#include <mechanisms/timer.h>
#include <opsystem/application_shell.h>
#include <opsystem/byte_filer.h>
#include <opsystem/filename.h>
#include <textual/list_parsing.h>

// the number of times we scan our data file for performance test.
const int MAX_DATA_FILE_ITERS = 4000;

#define LOG(s) EMERGENCY_LOG(program_wide_logger(), s)

class test_parsing_csv : public application_shell
{
public:
  test_parsing_csv();
  IMPLEMENT_CLASS_NAME("test_parsing_csv");
  int execute();
};

test_parsing_csv::test_parsing_csv()
: application_shell("t_parse_csv")
{
  SET_DEFAULT_COMBO_LOGGER;
}

#define COMPLAIN_FIELD(list, index, value) \
  log(isprintf("field %d: ", index) + list[index]); \
  if (list[index] != value) { \
    deadly_error(class_name(), "comparison test", isprintf("field %d is " \
        "erroneous in ", index) + #list); \
  }

int test_parsing_csv::execute()
{
  istring line1 = "\"fupe\",\"snoorp\",\"lutem\",\"fipe\"";
  string_array fields1;
  bool works1 = list_parsing::parse_csv_line(line1, fields1);  
  if (!works1) 
    deadly_error(class_name(), "first test", "fails to parse");
LOG(isprintf("fields len now %d", fields1.length()));
  if (fields1.length() != 4)
    deadly_error(class_name(), "first test", "wrong count of strings found");
  COMPLAIN_FIELD(fields1, 0, "fupe");
  COMPLAIN_FIELD(fields1, 1, "snoorp");
  COMPLAIN_FIELD(fields1, 2, "lutem");
  COMPLAIN_FIELD(fields1, 3, "fipe");

  istring line2 = "fupe,\"snoorp\",lutem,\"fipe\"";
  string_array fields2;
  bool works2 = list_parsing::parse_csv_line(line2, fields2);  
  if (!works2) 
    deadly_error(class_name(), "second test", "fails to parse");
  if (fields2.length() != 4)
    deadly_error(class_name(), "second test", "wrong count of strings found");
  COMPLAIN_FIELD(fields2, 0, "fupe");
  COMPLAIN_FIELD(fields2, 1, "snoorp");
  COMPLAIN_FIELD(fields2, 2, "lutem");
  COMPLAIN_FIELD(fields2, 3, "fipe");

  istring line3 = "\"lowenburger\",\"wazizzle\",morphel";
  string_array fields3;
  bool works3 = list_parsing::parse_csv_line(line3, fields3);  
  if (!works3) 
    deadly_error(class_name(), "third test", "fails to parse");
  if (fields3.length() != 3)
    deadly_error(class_name(), "third test", "wrong count of strings found");
  COMPLAIN_FIELD(fields3, 0, "lowenburger");
  COMPLAIN_FIELD(fields3, 1, "wazizzle");
  COMPLAIN_FIELD(fields3, 2, "morphel");

  istring line4 = "\"lowenburger\",\"wazizzle\",morphel,";
  string_array fields4;
  bool works4 = list_parsing::parse_csv_line(line4, fields4);  
  if (!works4) 
    deadly_error(class_name(), "fourth test", "fails to parse");
  if (fields4.length() != 4)
    deadly_error(class_name(), "fourth test", "wrong count of strings found");
  COMPLAIN_FIELD(fields4, 0, "lowenburger");
  COMPLAIN_FIELD(fields4, 1, "wazizzle");
  COMPLAIN_FIELD(fields4, 2, "morphel");
  COMPLAIN_FIELD(fields4, 3, "");

  istring line5 = "\"lowenburger\",,";
  string_array fields5;
  bool works5 = list_parsing::parse_csv_line(line5, fields5);  
  if (!works5) 
    deadly_error(class_name(), "fifth test", "fails to parse");
  if (fields5.length() != 3)
    deadly_error(class_name(), "fifth test", "wrong count of strings found");
  COMPLAIN_FIELD(fields5, 0, "lowenburger");
  COMPLAIN_FIELD(fields5, 1, "");
  COMPLAIN_FIELD(fields5, 2, "");

  istring line6 = ",,,\"rasputy\",,\"spunk\",ralph";
  string_array fields6;
  bool works6 = list_parsing::parse_csv_line(line6, fields6);  
  if (!works6) 
    deadly_error(class_name(), "sixth test", "fails to parse");
  if (fields6.length() != 7)
    deadly_error(class_name(), "sixth test", "wrong count of strings found");
  COMPLAIN_FIELD(fields6, 0, "");
  COMPLAIN_FIELD(fields6, 1, "");
  COMPLAIN_FIELD(fields6, 2, "");
  COMPLAIN_FIELD(fields6, 3, "rasputy");
  COMPLAIN_FIELD(fields6, 4, "");
  COMPLAIN_FIELD(fields6, 5, "spunk");
  COMPLAIN_FIELD(fields6, 6, "ralph");

  istring line7 = "\"SRV0001337CHN0000001DSP0000001SRV0001337LAY0003108,16,0,8,192\",\"\\\"row_3\\\" on 12.5.55.159\",3";
  string_array fields7;
  bool works7 = list_parsing::parse_csv_line(line7, fields7);
  if (!works7) 
    deadly_error(class_name(), "seventh test", "fails to parse");
  if (fields7.length() != 3)
    deadly_error(class_name(), "seventh test", "wrong count of strings found");
  COMPLAIN_FIELD(fields7, 0, "SRV0001337CHN0000001DSP0000001SRV0001337LAY0003108,16,0,8,192");
  COMPLAIN_FIELD(fields7, 1, "\"row_3\" on 12.5.55.159");
  COMPLAIN_FIELD(fields7, 2, "3");

  // test 8...  use data file.
  filename df_dir = filename(portable::application_name()).dirname();
  byte_filer test_data(df_dir.raw() + "/df_1.csv", "rb");
  string_array parsed;
  string_array lines;
  istring curr_line;
  int read_result;
  while ( (read_result = test_data.getline(curr_line, 1024)) > 0 )
    lines += curr_line;
  if (lines.length()) {
    // now we have the data file loaded.
    timer stopwatch;
    stopwatch.start();
    for (int iterations = 0; iterations < MAX_DATA_FILE_ITERS; iterations++) {
      for (int line = 0; line < lines.length(); line++) {
        const istring &current = lines[line];
        list_parsing::parse_csv_line(current, parsed);
      }
    }
    stopwatch.stop();
    log(isprintf("%d csv lines with %d iterations took %d ms (or %d s).",
        lines.length(), MAX_DATA_FILE_ITERS, stopwatch.elapsed(),
        stopwatch.elapsed() / 1000));
  }

  // test 9: process results of create_csv_line.
  string_array fields9;
  fields9 += "ACk\"boozort";
  fields9 += "sme\"ra\"\"foop";
  fields9 += "\"gumby\"";
  istring line9 = "\"ACk\\\"boozort\",\"sme\\\"ra\\\"\\\"foop\",\"\\\"gumby\\\"\"";
  istring gen_line_9;
  list_parsing::create_csv_line(fields9, gen_line_9);
//log(istring(" got gen line: ") + gen_line_9);
//log(istring("expected line: ") + line9);
  if (gen_line_9 != line9)
    deadly_error(class_name(), "ninth test", "failed to create expected text");
  string_array fields9_b;
  bool works9 = list_parsing::parse_csv_line(gen_line_9, fields9_b);
  if (!works9) 
    deadly_error(class_name(), "ninth test", "fails to parse");
  if (fields9_b != fields9)
    deadly_error(class_name(), "ninth test", "failed to match original fields");
  
  guards::alert_message("list_parsing::parse_csv_line: "
      "works for common examples.");

  return 0;
}


HOOPLE_MAIN(test_parsing_csv, )

---