t_tokenizer.cpp

Go to the documentation of this file.

/*****************************************************************************\
*                                                                             *
*  Name   : test_tokenizer                                                    *
*  Author : Chris Koeritz                                                     *
*                                                                             *
*  Purpose:                                                                   *
*                                                                             *
*    Puts the tokenizer through some paces.                                   *
*                                                                             *
*******************************************************************************
* Copyright (c) 1998-$now By Author.  This program is free software; you can  *
* redistribute it and/or modify it under the terms of the GNU General Public  *
* License as published by the Free Software Foundation; either version 2 of   *
* the License or (at your option) any later version.  This is online at:      *
*     http://www.fsf.org/copyleft/gpl.html                                    *
* Please send any updates to: fred@gruntose.com                               *
\*****************************************************************************/

#include <basis/byte_array.h>
#include <basis/function.h>
#include <basis/guards.h>
#include <basis/istring.h>
#include <opsystem/application_shell.h>
#include <opsystem/byte_filer.h>
#include <data_struct/static_memory_gremlin.h>
#include <data_struct/string_table.h>
#include <textual/parser_bits.h>
#include <textual/tokenizer.h>

HOOPLE_STARTUP_CODE;

#define TEST_OCM_STOCK_LIST
  // uncomment for a test similar to what the OCM does.

const int MAX_LINE_SIZE = 1000;
  // the largest line we will deal with in a file.

class test_tokenizer : public application_shell
{
public:
  test_tokenizer() : application_shell("t_tokenizer") {}

  IMPLEMENT_CLASS_NAME("test_tokenizer");

  virtual int execute();
};


int test_tokenizer::execute()
{
  {
    istring test_set_1 = "\n\
[frederick]\n\
samba=dance\n\
tantalus rex=gumby\n\
57 chevy heap=\"16 anagrams of misty immediately\"\n\
lingus distractus\n\
shouldus havus assignmentum=\n\
above better be parsed = 1\n\
;and this comment too yo\n\
ted=agent 12\n";

    const char *TEST = "First Test";
    istring testing = test_set_1;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed("\n\r", "=");
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    tokenizer gorp("\n\r", "=");
    gorp.parse(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());

//for (int i = 0; i < gorp.table().symbols(); i++) {
//istring name, value;
//gorp.table().retrieve(i, name, value);
//log(isprintf("item %d: name=\"%s\" value=\"%s\"", i, name.s(), value.s()));
//}

    if (!jed.exists("[frederick]"))
      deadly_error(class_name(), TEST, "jed section header was omitted!");
    if (jed.find("[frederick]") != "")
      deadly_error(class_name(), TEST, "jed section header had unexpected contents!");
    if (jed.find("ted") != "agent 12")
      deadly_error(class_name(), TEST, "jed's ted is missing or invalid!");
    if (jed.find("shouldus havus assignmentum").t())
      deadly_error(class_name(), TEST, "jed's shouldus had contents but shouldn't!");
    istring value = *jed.table().find("shouldus havus assignmentum");
    if (value != " ")
      deadly_error(class_name(), TEST, "shouldus had wrong contents, not special!");
    if (!gorp.exists("[frederick]"))
      deadly_error(class_name(), TEST, "gorp section header was omitted!");
    if (gorp.find("[frederick]") != "")
      deadly_error(class_name(), TEST, "gorp section header had unexpected contents!");
    if (gorp.find("ted") != "agent 12")
      deadly_error(class_name(), TEST, "gorp's ted is missing or invalid!");
    if (gorp.find("shouldus havus assignmentum").t())
      deadly_error(class_name(), TEST, "gorp's shouldus had contents but shouldn't!");
    value = *gorp.table().find("shouldus havus assignmentum");
    if (value != " ")
      deadly_error(class_name(), TEST, "shouldus had wrong contents, not special!");
  }    
  {
    istring test_set_2 = "Name=SRV,   Parent=,        Persist=Y,  Entry=Y, Required=Y, Desc=Server,               Tbl=Server";

    const char *TEST = "Second Test";
    istring testing = test_set_2;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed(",", "=");
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());
    if (jed.find("Name") != "SRV")
      deadly_error(class_name(), TEST, "Name is missing or invalid!");
    if (jed.find("Parent").t())
      deadly_error(class_name(), TEST, "Parent had contents but shouldn't!");
    istring value = *jed.table().find("Parent");
    if (value != " ")
      deadly_error(class_name(), TEST, "Parent had wrong contents, not special!");
    if (jed.find("Persist") != "Y")
      deadly_error(class_name(), TEST, "Persist is missing or invalid!");
  }    

//hmmm: this should be turned off when we don't have access to the file.
//      or...  why not just put a copy of the file in the test folder?
#ifdef TEST_OCM_STOCK_LIST
  {
    // test the tokenizer on the stock ocm list.  this is very specialized
    // and so is not part of the normal testing.
    byte_filer list_file("l:/packages/srvcfg/odms/ocm_list.ini", "r");
    istring ocms;
    byte_array chunks;
    istring the_line;
    int curr_line = 0;
    while (true) {
      chunks.reset();
      if (list_file.getline(chunks, MAX_LINE_SIZE)) {
        curr_line++;
        the_line = (char *)chunks.observe();
        // add the line before we strip anything, since the main tokenizer
        // needs to find CR on the end.
        ocms += the_line;
        // chop any extra spaces or returns off the end.
        while (the_line.length()
            && parser_bits::white_space(the_line[the_line.end()]))
          the_line.zap(the_line.end(), the_line.end());
        // eat each line and test tokenizer for a different set of parms.
        tokenizer liner(",", "=");
        liner.parse(the_line);
        log(istring(istring::SPRINTF, "%02d: ", curr_line) + liner.text_form());
      } else break;
    }

    tokenizer stocky("\n\r", "=");
    stocky.parse(ocms);
    log("");
    log("got these for the tokenized list:");
    log(stocky.text_form());
  }
#endif

  {
    istring test_set_3 = "\n\
[frederick]\n\
samba=dance\n\
tantalus rex=gumby \"don#t\n\n'play'\nthat\" homey '\n\ndog\n\n yo \"\ncreen\" arf'\n\
57 chevy heap=\"16 anagrams of misty immediately\"\n\
lingus distractus\n\
shouldus havus assignmentum=\n\
above better be parsed = 1\n\
;and this comment too yo\n\
ted=agent 12\n";

    const char *TEST = "Third Test";
    istring testing = test_set_3;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed("\n\r", "=", "\'\"");
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    tokenizer gorp("\n\r", "=", "\'\"");
    gorp.parse(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());
    if (!jed.exists("[frederick]"))
      deadly_error(class_name(), TEST, "section header was omitted!");
    if (jed.find("[frederick]") != "")
      deadly_error(class_name(), TEST, "section header had unexpected contents!");
    if (jed.find("ted") != "agent 12")
      deadly_error(class_name(), TEST, "ted is missing or invalid!");
    if (jed.find("shouldus havus assignmentum").t())
      deadly_error(class_name(), TEST, "shouldus had contents but shouldn't!");
    istring value = *jed.table().find("shouldus havus assignmentum");
    if (value != " ")
      deadly_error(class_name(), TEST, "shouldus had wrong contents, not special!");
    if (!gorp.exists("[frederick]"))
      deadly_error(class_name(), TEST, "section header was omitted!");
    if (gorp.find("[frederick]") != "")
      deadly_error(class_name(), TEST, "section header had unexpected contents!");
    if (gorp.find("ted") != "agent 12")
      deadly_error(class_name(), TEST, "ted is missing or invalid!");
    if (gorp.find("shouldus havus assignmentum").t())
      deadly_error(class_name(), TEST, "shouldus had contents but shouldn't!");
    value = *gorp.table().find("shouldus havus assignmentum");
    if (value != " ")
      deadly_error(class_name(), TEST, "shouldus had wrong contents, not special!");
    if (!gorp.exists("tantalus rex"))
      deadly_error(class_name(), TEST, "tantalus rex is missing!");
    if (gorp.find("tantalus rex")
        != "gumby \"don#t\n\n'play'\nthat\" homey '\n\ndog\n\n yo "
           "\"\ncreen\" arf'")
      deadly_error(class_name(), TEST, "tantalus rex has incorrect contents!");
  }
  {
    istring test_set_4 = "\n\
[garfola]\n\
treadmill=\"this ain't the place\nwhere'n we been done\nseein' no quotes\"\n\
borfulate='similarly \"we\" do not like\nthe \" quote \" type thing here'\n\
";

    const char *TEST = "Fourth Test";
    istring testing = test_set_4;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed("\n\r", "=", "\'\"", false);
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    tokenizer gorp("\n\r", "=", "\'\"", false);
    gorp.parse(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());
    if (!gorp.exists("[garfola]"))
      deadly_error(class_name(), TEST, "section header was omitted!");
    if (gorp.find("[garfola]") != "")
      deadly_error(class_name(), TEST, "section header had unexpected contents!");
    if (!gorp.exists("treadmill"))
      deadly_error(class_name(), TEST, "treadmill is missing!");
    if (gorp.find("treadmill")
        != "\"this ain't the place\nwhere'n we been done\nseein' no quotes\"")
      deadly_error(class_name(), TEST, "treadmill has incorrect contents!");
    if (!gorp.exists("borfulate"))
      deadly_error(class_name(), TEST, "borfulate is missing!");
    if (gorp.find("borfulate")
        != "'similarly \"we\" do not like\nthe \" quote \" type thing here'")
      deadly_error(class_name(), TEST, "borfulate has incorrect contents!");
  }
  {
    istring test_set_5 = "\n\
 x~35; y~92   ;#comment   ; d   ~83 ;  e~   54   ; ? new comment  ;sud   ~  xj23-8 ; nigh ~2";

    const char *TEST = "Fifth Test";
    istring testing = test_set_5;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed(";", "~");
    jed.set_comment_chars("#?");
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());

    tokenizer gorp(";", "~");
    gorp.set_comment_chars("#?");
    gorp.parse(out);
    log("gorp in tabular form:");
    log(gorp.table().text_form());
    if (gorp.table() != jed.table()) 
      deadly_error(class_name(), TEST, "gorp text not same as jed!");

    if (jed.find("x") != "35")
      deadly_error(class_name(), TEST, "value for x missing or invalid");
    if (jed.find("y") != "92")
      deadly_error(class_name(), TEST, "value for y missing or invalid");
    if (jed.find("d") != "83")
      deadly_error(class_name(), TEST, "value for d missing or invalid");
    if (jed.find("e") != "54")
      deadly_error(class_name(), TEST, "value for e missing or invalid");
    if (jed.find("sud") != "xj23-8")
      deadly_error(class_name(), TEST, "value for sud missing or invalid");
    if (jed.find("nigh") != "2")
      deadly_error(class_name(), TEST, "value for nigh missing or invalid");
  }    
  {
    istring test_set_6 = "\r\n\r\n\r\
# this is yet another test with comments.\r\n\
; we want to be sure stuff works right.\r\n\
crumpet=tempest\r\n\
  moomar=18\r\n\
shagbot  =once upon a time there was a man  \r\n\
\t\t\tpunzola megamum  =brandle the handle  \r\n\
trapzoot=  uhhh\r\n\
mensch   = racer X\r\n\
\r\n\r\n\r\n";

    const char *TEST = "Sixth Test";
    istring testing = test_set_6;
    log(istring("file before parsing:\n") + testing);
    tokenizer jed("\n\r", "=");
    jed.set_comment_chars("#;");
    jed.parse(testing);
    istring out;
    jed.text_form(out);
    log(istring("file after parsing:\n") + out);
    log("and in tabular form:");
    log(jed.table().text_form());

    tokenizer gorp("\n\r", "=");
    gorp.set_comment_chars("#;");
    gorp.parse(out);
    log("gorp in tabular form:");
    log(gorp.table().text_form());
log(isprintf("gorp has %d fields, jed has %d fields", gorp.symbols(), jed.symbols()));
    if (gorp.table() != jed.table()) 
      deadly_error(class_name(), TEST, "gorp text not same as jed!");

    if (jed.find("crumpet") != "tempest")
      deadly_error(class_name(), TEST, "value for crumpet missing or invalid");
    if (jed.find("moomar") != "18")
      deadly_error(class_name(), TEST, "value for moomar missing or invalid");
    if (jed.find("shagbot") != "once upon a time there was a man")
      deadly_error(class_name(), TEST, "value for shagbot missing or invalid");
    if (jed.find("trapzoot") != "uhhh")
      deadly_error(class_name(), TEST, "value for trapzoot missing or invalid");
    if (jed.find("punzola megamum") != "brandle the handle")
      deadly_error(class_name(), TEST, "value for punzola missing or invalid");
    if (jed.find("mensch") != "racer X")
      deadly_error(class_name(), TEST, "value for mensch missing or invalid");
  }    

  guards::alert_message("tokenizer:: works for those functions tested.");
  return 0;
}


int main(int formal(argc), char *formal(argv)[])
{
  test_tokenizer to_test;
  return to_test.execute();
}

---