link_parser.cpp

Go to the documentation of this file.

/*****************************************************************************\
*                                                                             *
*  Name   : link_parser                                                       *
*  Author : Chris Koeritz                                                     *
*                                                                             *
*  Purpose:                                                                   *
*                                                                             *
*    Processes html files and finds the links.  A database in the HOOPLE      *
*  link format is created from the links found.                               *
*                                                                             *
*******************************************************************************
* Copyright (c) 1991-$now By Author.  This program is free software; you can  *
* redistribute it and/or modify it under the terms of the GNU General Public  *
* License as published by the Free Software Foundation; either version 2 of   *
* the License or (at your option) any later version.  This is online at:      *
*     http://www.fsf.org/copyleft/gpl.html                                    *
* Please send any updates to: fred@gruntose.com                               *
\*****************************************************************************/

// Notes:
//
// the standard link structure in html is similar to this:
//     <a href="blahblah">Link Name and Launching Point</a>
//
// the standard we adopt for section titles is that it must be a heading
// marker.  that formatting looks like this, for example:
//     <h3 assorted_stuff>The Section Title:</h3>

#include <basis/function.h>
#include <basis/guards.h>
#include <basis/istring.h>
#include <opsystem/application_shell.h>
#include <opsystem/byte_filer.h>
#include <opsystem/command_line.h>
#include <loggers/file_logger.h>
#include <opsystem/filename.h>
#include <data_struct/static_memory_gremlin.h>
#include <textual/parser_bits.h>

#undef BASE_LOG
#define BASE_LOG(s) program_wide_logger().log(s)
#undef LOG
#define LOG(s) CLASS_EMERGENCY_LOG(program_wide_logger(), s)

const int MAX_FILE_SIZE = 4 * MEGABYTE;
  // this is the largest html file size we will process.


// a macro that increments the position in the string and restarts the loop.
#define INCREM_N_GO { curr_index++; continue; }

// puts the current character on the intermediate string.
#define ADD_INTERMEDIATE \
  intermediate_text += full_contents[curr_index]

bool caseless_equals(char to_find, char comparing_with)
{
  if (to_find == comparing_with) return true;
  if ( (to_find >= 'a') && (to_find <= 'z') 
      && (to_find - ('a' - 'A') == comparing_with) ) return true;
  return false;
}

// a macro that skips all characters until the specified one is seen.
#define JUMP_TO_CHAR(to_find, save_them) { \
  while ( (curr_index < full_contents.length()) \
      && !caseless_equals(to_find, full_contents[curr_index]) ) { \
    if (save_them) ADD_INTERMEDIATE; \
    curr_index++; \
  } \
}

// increments the state, the current character and restarts the loop.
#define NEXT_STATE_INCREM { \
  state = parsing_states(state+1);  /* move forward in states. */ \
  curr_index++; \
  continue; \
}

// cleans out the disallowed characters in the string provided.
#define CLEAN_UP_NAUGHTY(s) { \
  while (s.replace("\n", " ")) {} \
  while (s.replace("\r", "")) {} \
  int indy = s.find("--"); \
  while (non_negative(indy)) { \
    s[indy] = ' ';  /* replace the first dash with a space. */ \
    for (int i = indy + 1; i < s.length(); i++) { \
      if (s[i] != '-') break; \
      s.zap(i, i); \
      i--; \
    } \
    indy = s.find("--"); \
  } \
  while (s.replace("  ", " ")) {} \
  s.strip_spaces(); \
}

// cleans up underscores in areas that are supposed to be english.
#define MAKE_MORE_ENGLISH(s) \
  s.replace_all('_', ' ')

void strain_out_html_codes(istring &to_edit)
{
  for (int i = 0; i < to_edit.length(); i++) {
    if (to_edit[i] != '<') continue;
    // found a left bracket.
    int indy = to_edit.find('>', i);
    if (negative(indy)) return;  // bail out, unexpected unmatched bracket.
    to_edit.zap(i, indy);
    i--;  // skip back to reconsider current place.
  }
}

// writes out the currently accumulated link info.
#define WRITE_LINK { \
  /* clean naughty characters out of the names. */ \
  CLEAN_UP_NAUGHTY(url_string); \
  CLEAN_UP_NAUGHTY(name_string); \
  while (name_string.replace("\n", " ")) {} \
  while (name_string.replace("\r", "")) {} \
  if (url_string.ends(name_string)) { \
    /* handle the name being boring. replace with the intermediate text. */ \
    MAKE_MORE_ENGLISH(intermediate_text); \
    strain_out_html_codes(intermediate_text); \
    CLEAN_UP_NAUGHTY(intermediate_text); \
    if (intermediate_text.length()) \
      name_string = intermediate_text; \
  } \
  /* output a link in the HOOPLE format. */ \
  istring to_write = "\"L\",\""; \
  to_write += name_string; \
  to_write += "\",\""; \
  to_write += last_title; \
  to_write += "\",\""; \
  to_write += url_string; \
  to_write += "\"\n"; \
  output_file.write(to_write); \
  _link_count++; \
}

// writes out the current section in the HOOPLE format.
// currently the parent category is set to NOP.
#define WRITE_SECTION { \
  CLEAN_UP_NAUGHTY(last_title);  /* clean the name. */ \
  /* output a category definition. */ \
  istring to_write = "\"C\",\""; \
  to_write += last_title; \
  to_write += "\",\""; \
  to_write += "NOP"; \
  to_write += "\"\n"; \
  output_file.write(to_write); \
  _category_count++; \
}

// clears our accumulator strings.
#define RESET_STRINGS { \
  url_string = istring::empty_string(); \
  name_string = istring::empty_string(); \
  intermediate_text = istring::empty_string(); \
}


class link_parser : public application_shell
{
public:
  link_parser() : application_shell(static_class_name()), _link_count(0),
      _category_count(0) {}
  IMPLEMENT_CLASS_NAME("link_parser");
  virtual int execute();
  int print_instructions(const filename &program_name);

private:
  int _link_count;  // number of links.
  int _category_count;  // number of categories.

  istring url_string;  // the URL we've parsed.
  istring name_string;  // the name that we've parsed for the URL.
  istring last_title;  // the last name that was set for a section.
  istring intermediate_text;  // strings we saw before a link.

  istring heading_num;
    // this string form of a number tracks what kind of heading was started.
};


int link_parser::print_instructions(const filename &program_name)
{
  isprintf to_show("%s:\n\
This program needs two filenames as command line parameters.  The -i flag\n\
is used to specify the input filename and the -o flag specifies the output\n\
file to be created.  The input file is expected to be an html file\n\
containing links to assorted web sites.  The links are gathered, along with\n\
descriptive text that happens to be near them, to create a link database in\n\
the HOOPLE link format and write it to the output file.  HOOPLE link format\n\
is basically a CSV file that defines the columns 1-4 for describing either\n\
link categories (which support hierarchies) or actual links (i.e., URLs of\n\
interest).  The links are written to a CSV file in the standard HOOPLE link\n\
The HOOPLE link format is documented here:\n\
    http://hoople.org/guides/link_database/format_manifesto.txt\n\
", program_name.basename().raw().s(), program_name.basename().raw().s());
  program_wide_logger().log(to_show.s());
  return 12;
}

int link_parser::execute()
{
  FUNCDEF("main");
  SET_DEFAULT_COMBO_LOGGER;

  command_line cmds(__argc, __argv);  // process the command line parameters.
  istring input_filename;  // we'll store our bookmarks file's name here.
  istring output_filename;  // where the processed marks go.
  if (!cmds.get_value('i', input_filename, false))
    return print_instructions(cmds.program_name());
  if (!cmds.get_value('o', output_filename, false))
    return print_instructions(cmds.program_name());

  BASE_LOG(istring("input file: ") + input_filename);
  BASE_LOG(istring("output file: ") + output_filename);

  istring full_contents;
  byte_filer input_file(input_filename, "r");
  if (!input_file.good())
    non_continuable_error(class_name(), func, "the input file could not be opened");
  input_file.read(full_contents, MAX_FILE_SIZE);
  input_file.close();

  filename outname(output_filename);
  if (outname.exists()) {
    non_continuable_error(class_name(), func, istring("the output file ")
        + output_filename + " already exists.  It would be over-written if "
        "we continued.");
  }

  byte_filer output_file(output_filename, "w");
  if (!output_file.good())
    non_continuable_error(class_name(), func, "the output file could not be opened");

  enum parsing_states {
    // the states below are order dependent; do not change the ordering!
    SEEKING_LINK_START,  // looking for the beginning of an html link.
    SEEKING_HREF,  // finding the href portion of the link.
    GETTING_URL,  // chowing on the URL portion of the link.
    SEEKING_NAME,  // finding the closing bracket of the <a ...>.
    GETTING_NAME,  // chowing down on characters in the link's name.
    SEEKING_CLOSURE,  // looking for the </a> to end the link.
    // there is a discontinuity after SEEKING_CLOSURE, but then the following
    // states are also order dependent.
    SAW_TITLE_START,  // the beginning of a section heading was seen.
    GETTING_TITLE  // grabbing characters in the title.
  };

  int curr_index = 0;
  parsing_states state = SEEKING_LINK_START;
  while (curr_index < full_contents.length()) {
    switch (state) {
      case SEEKING_LINK_START:
        // if we don't see a less-than, then it's not the start of html code,
        // so we'll ignore it for now.
        if (full_contents[curr_index] != '<') {
          ADD_INTERMEDIATE;
          INCREM_N_GO;
        }
        // found a left angle bracket, so now we need to make sure this is
        // an address style code.
        curr_index++;
        if (caseless_equals('h', full_contents[curr_index])) {
          // check that we're seeing a heading definition here.
          const char next = full_contents[curr_index + 1];
          if ( (next >= '0') && (next <= '9') ) {
            // we found our proper character for starting a heading.  we need
            // to jump into that state now.  we'll leave the cursor at the
            // beginning of the number.
            state = SAW_TITLE_START;
            INCREM_N_GO;
          }
        }
        if (!caseless_equals('a', full_contents[curr_index])) {
          intermediate_text += '<';
          JUMP_TO_CHAR('>', true);
          continue; 
        }
        // found an a, but make sure that's the only character in the word.
        curr_index++;
        if (!parser_bits::white_space(full_contents[curr_index])) {
          intermediate_text += "<a";
          JUMP_TO_CHAR('>', true);
          continue; 
        }
        // this looks like an address so find the start of the href.
        NEXT_STATE_INCREM;
        break;
      case SEEKING_HREF:
        JUMP_TO_CHAR('h', false);  // find the next 'h' for "href".
        curr_index++;
        if (!caseless_equals('r', full_contents[curr_index])) continue;
        curr_index++;
        if (!caseless_equals('e', full_contents[curr_index])) continue;
        curr_index++;
        if (!caseless_equals('f', full_contents[curr_index])) continue;
        curr_index++;
        if (full_contents[curr_index] != '=') continue;
        curr_index++;
        if (full_contents[curr_index] != '"') continue;
        // whew, got through the word href and the assignment.  the rest
        // should all be part of the link.
        NEXT_STATE_INCREM;
        break;
      case GETTING_URL:
        // as long as we don't see the closure of the quoted string for the
        // href, then we can keep accumulating characters from it.
        if (full_contents[curr_index] == '"') NEXT_STATE_INCREM;
        url_string += full_contents[curr_index];
        INCREM_N_GO;  // keep chewing on it in this same state.
        break;
      case SEEKING_NAME:
        JUMP_TO_CHAR('>', false);  // find closing bracket.
        NEXT_STATE_INCREM;  // now start grabbing the name characters.
        break;
      case GETTING_NAME:
        // we have to stop grabbing name characters when we spy a new code
        // being started.
        if (full_contents[curr_index] == '<') {
          // if we see a closing command, then we assume it's the one we want.
          if (full_contents[curr_index + 1] == '/')
            NEXT_STATE_INCREM;
          // if we see html inside the name, we just throw it out.
          JUMP_TO_CHAR('>', false);
          curr_index++;
          continue;
        }
        name_string += full_contents[curr_index];
        INCREM_N_GO;  // keep chewing on it in this same state.
        break;
      case SEEKING_CLOSURE:
        JUMP_TO_CHAR('>', false);  // find the closure of the html code.
        // write the link out now.
        WRITE_LINK;
        // clean out our accumulated strings.
        RESET_STRINGS;
        state = SEEKING_LINK_START;
        INCREM_N_GO;
        break;
      case SAW_TITLE_START:
        heading_num = full_contents.substring(curr_index, curr_index);
        JUMP_TO_CHAR('>', false);
        NEXT_STATE_INCREM;  // start eating the name.
        break;
      case GETTING_TITLE: {
        int indy = full_contents.find('<', curr_index);
        if (negative(indy)) {
          state = SEEKING_LINK_START;  // too weird, go back to start.
          continue;
        }
        last_title = full_contents.substring(curr_index, indy - 1);
        WRITE_SECTION;
        JUMP_TO_CHAR('<', false);  // now find the start of the header closure.
        JUMP_TO_CHAR('>', false);  // now find the end of the header closure.
        RESET_STRINGS;
        state = SEEKING_LINK_START;  // successfully found section name.
        break;
      }
      default:
        non_continuable_error(class_name(), func, "entered erroneous state!");
    }
  }

  if (url_string.t()) WRITE_LINK;

  output_file.close();

  BASE_LOG(isprintf("wrote %d links in %d categories.", _link_count,
      _category_count));

  return 0;
}


HOOPLE_MAIN(link_parser, )

#ifdef __BUILD_STATIC_APPLICATION__
  // static dependencies found by buildor_gen_deps.sh:
  #include <basis/array.cpp>
  #include <basis/byte_array.cpp>
  #include <basis/callstack_tracker.cpp>
  #include <basis/chaos.cpp>
  #include <basis/convert_utf.cpp>
  #include <basis/definitions.cpp>
  #include <basis/earth_time.cpp>
  #include <basis/guards.cpp>
  #include <basis/istring.cpp>
  #include <basis/log_base.cpp>
  #include <basis/memory_checker.cpp>
  #include <basis/mutex.cpp>
  #include <basis/object_base.cpp>
  #include <basis/outcome.cpp>
  #include <basis/packable.cpp>
  #include <basis/portable.cpp>
  #include <basis/sequence.cpp>
  #include <basis/set.cpp>
  #include <basis/utility.cpp>
  #include <basis/version_record.cpp>
  #include <data_struct/amorph.cpp>
  #include <data_struct/bit_vector.cpp>
  #include <data_struct/byte_hasher.cpp>
  #include <data_struct/configurator.cpp>
  #include <data_struct/hash_table.cpp>
  #include <data_struct/pointer_hash.cpp>
  #include <data_struct/stack.cpp>
  #include <data_struct/static_memory_gremlin.cpp>
  #include <data_struct/string_hash.cpp>
  #include <data_struct/string_hasher.cpp>
  #include <data_struct/string_table.cpp>
  #include <data_struct/symbol_table.cpp>
  #include <data_struct/table_configurator.cpp>
  #include <loggers/console_logger.cpp>
  #include <loggers/file_logger.cpp>
  #include <loggers/locked_logger.cpp>
  #include <loggers/null_logger.cpp>
  #include <loggers/program_wide_logger.cpp>
  #include <opsystem/application_base.cpp>
  #include <opsystem/application_shell.cpp>
  #include <opsystem/byte_filer.cpp>
  #include <opsystem/command_line.cpp>
  #include <opsystem/critical_events.cpp>
  #include <opsystem/directory.cpp>
  #include <opsystem/filename.cpp>
  #include <opsystem/ini_config.cpp>
  #include <opsystem/ini_parser.cpp>
  #include <opsystem/path_configuration.cpp>
  #include <opsystem/rendezvous.cpp>
  #include <textual/byte_format.cpp>
  #include <textual/parser_bits.cpp>
  #include <textual/string_manipulation.cpp>
  #include <textual/tokenizer.cpp>
#endif // __BUILD_STATIC_APPLICATION__

---