bundle_creator.cpp

Go to the documentation of this file.

//hmmm: anything related to _stub_size should be kept, but that is where
//      we need a redundant search mechanism that can't be fooled so easily
//      by modifying exe; make a pattern that will be found and is the first
//      place to start looking for manifest.

/*****************************************************************************\
*                                                                             *
*  Name   : bundle_creator                                                    *
*  Author : Chris Koeritz                                                     *
*                                                                             *
*******************************************************************************
* Copyright (c) 2006-$now By Author.  This program is free software; you can  *
* redistribute it and/or modify it under the terms of the GNU General Public  *
* License as published by the Free Software Foundation; either version 2 of   *
* the License or (at your option) any later version.  This is online at:      *
*     http://www.fsf.org/copyleft/gpl.html                                    *
* Please send any updates to: fred@gruntose.com                               *
\*****************************************************************************/

#include "common_bundle.h"

#include <basis/array.cpp>
#include <basis/byte_array.h>
#include <basis/portable.h>
#include <data_struct/string_table.h>
#include <opsystem/application_shell.h>
#include <opsystem/byte_filer.h>
#include <opsystem/command_line.h>
#include <loggers/console_logger.h>
#include <opsystem/directory.h>
#include <opsystem/filename.h>
#include <loggers/file_logger.h>
#include <opsystem/filetime.h>
#include <opsystem/ini_config.h>
#include <opsystem/path_configuration.h>
#include <data_struct/static_memory_gremlin.h>
#include <textual/byte_format.h>
#include <textual/list_parsing.h>
#include <textual/parser_bits.h>
#include <textual/tokenizer.h>

#include <stdio.h>
#include <sys/stat.h>
#include <zlib.h>
#ifdef __WIN32__
  #include <io.h>
#endif

const int CHUNKING_SIZE = 256 * KILOBYTE;
  // we'll read this big a chunk from a source file at a time.

#define BASE_LOG(to_print) program_wide_logger().log(to_print)
#define LOG(to_print) CLASS_EMERGENCY_LOG(program_wide_logger(), to_print)

//#define DEBUG_BUNDLER
  // uncomment for noisy debugging version.

// returns the "retval" and mentions that this is a failure at "where".
#define FAIL_RETURN(retval, where) { \
  LOG(istring("failure in ") + where + isprintf(", exit=%d", retval)); \
  return retval; \
}


inline bool true_value(const istring &value)
{ return (value != "0") && (value != "false"); }


// this structure overrides the manifest_chunk by providing a source string.

struct bundled_chunk : manifest_chunk
{
  istring _source;  
  virtual ~bundled_chunk() {}
};


// main bundler class.

class bundle_creator : public application_shell
{
public:
  bundle_creator()
      : application_shell(static_class_name()),
        _app_name(filename(__argv[0]).basename()),
        _bundle(NIL), _stub_size(0), _keyword() {}

  virtual ~bundle_creator() {
    WHACK(_bundle);
  }

  IMPLEMENT_CLASS_NAME("bundle_creator");
  virtual int execute();
  int print_instructions();

  int open_output_file();

  int read_manifest();

  int write_stub_and_toc();

  int bundle_sources();

  int finalize_file();

  int write_offset();

  int patch_recursive_target(const istring &source, const istring &target,
          int manifest_index);

  int recurse_into_dir(const istring &source, const istring &target,
          int manifest_index);

  int patch_wildcard_target(const istring &source, const istring &target,
          int manifest_index);

  int add_files_here(directory &dirndl, const istring &source,
          const istring &target, int manifest_index);

  bool get_file_size(const istring &file, int &size, byte_array &timestamp);

private:
  istring _app_name;  
  istring _output_file;  
  istring _manifest_file;  
  array<bundled_chunk> _manifest_list;  
  byte_filer *_bundle;  
  int _stub_size;  
  istring _keyword;  // set if we were given a keyword on cmd line.
};


int bundle_creator::print_instructions()
{
  BASE_LOG(isprintf("\
%s: This program needs two parameters on the command line.\n\
The -o flag must point at the bundled output file to create.  The -m flag\n\
must point at a valid manifest file that defines what will be packed into\n\
the output file.  See the example manifest file for more information on\n\
the required file format.\n\
", _app_name.s()));
  return 4;
}

int bundle_creator::execute()
{
  FUNCDEF("execute");

  BASE_LOG(istring("starting file bundling at ") + timestamp(false, true));

  command_line cmds(__argc, __argv);
  istring temp;
  if (cmds.get_value('?', temp)) return print_instructions();
  if (cmds.get_value("?", temp)) return print_instructions();
  if (!cmds.get_value('o', _output_file)) return print_instructions();
  if (!cmds.get_value('m', _manifest_file)) return print_instructions();

  if (filename(_output_file).exists()) {
    BASE_LOG(isprintf("\
%s: The output file already exists.  Please move it out of\n\
the way; this program will not overwrite existing files.\n",
_app_name.s()));
    return 3;
  }

  if (!filename(_manifest_file).exists()) {
    BASE_LOG(isprintf("\
%s: The manifest file does not exist.  This program cannot do anything\n\
without a valid packing manifest.\n", _app_name.s()));
    return 2;
  }

  // make sure we snag any keyword that was passed on the command line.
  cmds.get_value("keyword", _keyword);

  // first step is to provide some built-in variables that can be used to
  // make the manifests less platform specific.  this doesn't really help
  // if you bundle it on linux and try to run it on windows.  but either
  // platform's resources can easily be made into a bundle with the same
  // packing manifest.
#ifndef __WIN32__
  portable::set_environ("EXE_END", "");  // executable file ending.
  portable::set_environ("DLL_START", "lib");  // dll file prefix.
  portable::set_environ("DLL_END", ".so");  // dll file ending.
#else
  portable::set_environ("EXE_END", ".exe");
  portable::set_environ("DLL_START", "");
  portable::set_environ("DLL_END", ".dll");
#endif

  int ret = 0;
  if ( (ret = read_manifest()) ) FAIL_RETURN(ret, "reading manifest");
    // read manifest to build list of what's what.
  if ( (ret = open_output_file()) ) FAIL_RETURN(ret, "opening output file");
    // open up our output file for the bundled chunks.
  if ( (ret = write_stub_and_toc()) ) FAIL_RETURN(ret, "writing stub and TOC");
    // writes the stub unpacker application and the table of contents to the 
    // output file.
  if ( (ret = bundle_sources()) ) FAIL_RETURN(ret, "bundling source files");
    // stuff all the source files into the output bundle.
  if ( (ret = finalize_file()) ) FAIL_RETURN(ret, "finalizing file");
    // finishes with the file and closes it up.
  if ( (ret = write_offset()) ) FAIL_RETURN(ret, "writing offset");
    // stores the offset of the TOC into the output file in a special location
    // that is delineated by a known keyword (muftiloc) and which should only
    // exist in the file in one location.

  return 0;
}

int bundle_creator::open_output_file()
{
  FUNCDEF("open_output_file");
  _bundle = new byte_filer(_output_file, "wb");
  if (!_bundle->good()) {
    LOG(istring("failed to open the output file: ") + _output_file);
    return 65;
  }
  return 0;
}

bool bundle_creator::get_file_size(const istring &infile, int &size,
    byte_array &time_stamp)
{
  FUNCDEF("get_file_size");
  time_stamp.reset();
  // access the source file to get its size.
  byte_filer source_file(infile, "rb");
  if (!source_file.good()) {
    LOG(istring("could not access the file for size check: ") + infile);
    return false;
  }
  size = int(source_file.length());
  file_time tim(infile);
  tim.pack(time_stamp);
  return true;
}

int bundle_creator::add_files_here(directory &dirndl, const istring &source,
    const istring &target, int manifest_index)
{
  FUNCDEF("add_files_here");
  for (int i = 0; i < dirndl.files().length(); i++) {
    istring s = dirndl.files()[i];
//LOG(istring("file is: ") + s);
    bundled_chunk new_guy;
    new_guy._source = source + "/" + s;  // the original full path to it.
    new_guy._target = target + "/" + s;
    new_guy._keywords = _manifest_list[manifest_index]._keywords;

//LOG(isprintf("adding: source=%s targ=%s", new_guy._source.s(), new_guy._target.s()));
    bool okaysize = get_file_size(new_guy._source, new_guy._size,
        new_guy._timestamp);
    if (!okaysize || (new_guy._size < 0) ) {
      LOG(istring("failed to get file size for ") + new_guy._source);
      return 75;
    }

    _manifest_list.insert(manifest_index + 1, 1);
    _manifest_list[manifest_index + 1] = new_guy;
  }
  return 0;
}

int bundle_creator::recurse_into_dir(const istring &source,
    const istring &target, int manifest_index)
{
  FUNCDEF("recurse_into_dir");
//LOG(istring("src=") + source + " dest=" + target);

  string_array dirs;  // culled from the directory listing.
  {
    // don't pay for the directory object on the recursive invocation stack;
    // just have what we need on the stack (the directory list).
    directory dirndl(source);
//check dir for goodness!
    int ret = add_files_here(dirndl, source, target, manifest_index);
      // add in just the files that were found.
    if (ret != 0) {
      // this is a failure, but the function complains about it already.
      return 75;
    }
    dirs = dirndl.directories();
  }

//LOG("now scanning directories...");

  // now scan across the directories we found.
  for (int i = 0; i < dirs.length(); i++) {
    istring s = dirs[i];
//LOG(istring("curr dir is ") + s);
    int ret = recurse_into_dir(source + "/" + s, target + "/"
        + s, manifest_index);
    if (ret != 0) return ret;  // bail out.
  }

  return 0;
}

int bundle_creator::patch_recursive_target(const istring &source,
    const istring &target, int manifest_index)
{
  FUNCDEF("patch_recursive_target");
//LOG(istring("patch recurs src=") + source + " targ=" + target);
  return recurse_into_dir(source, target, manifest_index);
}

int bundle_creator::patch_wildcard_target(const istring &source,
    const istring &target, int manifest_index)
{
  FUNCDEF("patch_wildcard_target");
  // find the last slash.  the rest is our wildcard component.
  int src_end = source.end();
  int slash_indy = source.find('/', src_end, true);
  istring real_source = source.substring(0, slash_indy - 1);
  istring wild_pat = source.substring(slash_indy + 1, src_end);
//BASE_LOG(istring("got src=") + real_source + " wildpat=" + wild_pat);

  directory dirndl(real_source, wild_pat.s());
//check dir for goodness!
  int ret = add_files_here(dirndl, real_source, target, manifest_index);
  if (ret != 0) {
    // this is a failure, but the function complains about it already.
    return 75;
  }

  return 0;
}

int bundle_creator::read_manifest()
{
  FUNCDEF("read_manifest");
  ini_configurator ini(_manifest_file, configurator::RETURN_ONLY);
  string_table toc;
  bool worked = ini.get_section("toc", toc);
  if (!worked) {
    LOG(istring("failed to read TOC section in manifest:\n") + _manifest_file
        + "\ndoes that file exist?");
    return 65;
  }

//hmmm: make a class member.
  file_logger noisy_logfile(path_configuration::make_logfile_name
      ("bundle_creator_activity.log"));
  noisy_logfile.log(istring('-', 76));
  noisy_logfile.log(istring("Bundling starts at ") + timestamp(false, true));

  // add enough items in the list for our number of sections.
  _manifest_list.insert(0, toc.symbols());
  istring value;  // temporary string used below.
  int final_return = 0;  // if non-zero, an error occurred.

#define BAIL(retval) \
  final_return = retval; \
  toc.zap_index(i); \
  _manifest_list.zap(i, i); \
  i--; \
  continue

  for (int i = 0; i < toc.symbols(); i++) {
    // read all the info in this section and store it into our list.
    istring section_name = toc.name(i);
    section_name.strip_spaces(istring::FROM_FRONT);
    if (section_name[0] == '#') {
//hmmm: this looks a bit familiar from bail macro above.  abstract out?
      toc.zap_index(i);
      _manifest_list.zap(i, i);
      i--;
      continue;  // skip comments.
    }

    // check for any keywords on the section.  these are still needed for
    // variables, which otherwise would skip the rest of the field checks.
    if (ini.get(section_name, "keyword", value)) {
      string_array keys;
      bool worked = list_parsing::parse_csv_line(value, keys);
      if (!worked) {
        LOG(istring("failed to parse keywords for section ")
            + section_name + " in " + _manifest_file);
        BAIL(82);
      }
      _manifest_list[i]._keywords = keys;
      istring dumped;
      list_parsing::create_csv_line(_manifest_list[i]._keywords, dumped);
      noisy_logfile.log(section_name + " keywords: " + dumped);
    }

    if (ini.get(section_name, "variable", value)) {
      // this is a variable assignment.  it is the only thing we care about
      // for this section, so the rest is ignored.
      tokenizer zohre;
      zohre.parse(value);
      if (zohre.symbols() < 1) {
        LOG(istring("failed to parse a variable statement from ") + value);
        BAIL(37);
      }
      _manifest_list[i]._flags = SET_VARIABLE;  // not orred, just this.
      // set the two parts of our variable.
      _manifest_list[i]._target = zohre.table().name(0);
      _manifest_list[i]._parms = zohre.table()[0];
      BASE_LOG(istring("will set ") + _manifest_list[i]._target + " = "
          + _manifest_list[i]._parms);
//hmmm: make this optional in the future.
      portable::set_environ(_manifest_list[i]._target,
          parser_bits::substitute_env_vars(_manifest_list[i]._parms));
      continue;
    }

    if (!ini.get(section_name, "source", _manifest_list[i]._source)) {
      // check whether they told us not to pack and it's executable.
      bool okay_to_omit_source = false;
      istring value2;
      if (ini.get(section_name, "no_pack", value)
          && ini.get(section_name, "exec_target", value2) ) {
        if (true_value(value) && true_value(value2)) {
          // this type of section doesn't need source declared.
          okay_to_omit_source = true;
        }
      }
      if (!okay_to_omit_source) {
        LOG(istring("failed to read the source entry for section ")
            + section_name + " in " + _manifest_file);
        BAIL(67);
      }
    }
    // fix meshugener backslashes so we can count on the slash direction.
    _manifest_list[i]._source.replace_all('\\', '/');

    if (!ini.get(section_name, "target", _manifest_list[i]._target)) {
      // check whether they told us not to pack and it's executable.
      bool okay_to_omit_target = false;
      istring value2;
      if (ini.get(section_name, "no_pack", value)
          && ini.get(section_name, "exec_source", value2) ) {
        if (true_value(value) && true_value(value2)) {
          // this type of section doesn't need target declared.
          okay_to_omit_target = true;
        }
      }
      if (!okay_to_omit_target) {
        LOG(istring("failed to read the target entry for section ")
            + section_name + " in " + _manifest_file);
        BAIL(68);
      }
    }
    // fix backslashes in target also.
    _manifest_list[i]._target.replace_all('\\', '/');

    // capture any parameters they have specified for exec or other options.
    if (ini.get(section_name, "parms", value)) {
      _manifest_list[i]._parms = value;
#ifdef DEBUG_BUNDLER
      BASE_LOG(istring("got parms for ") + section_name + " as: " + value);
#endif
      if (value[0] != '"') {
        // repair the string if we're running on windows.
        _manifest_list[i]._parms = istring("\"") + value + "\"";
      }
      noisy_logfile.log(section_name + " parms: " + _manifest_list[i]._parms);
    }

    // check for the ignore errors flag.
    if (ini.get(section_name, "error_okay", value)) {
      if (true_value(value))
        _manifest_list[i]._flags |= IGNORE_ERRORS;
    }

    // see if they are saying not to overwrite the target file.
    if (ini.get(section_name, "no_replace", value)) {
      if (true_value(value))
        _manifest_list[i]._flags |= NO_OVERWRITE;
    }

    // see if they are saying not to overwrite the target file.
    if (ini.get(section_name, "quiet", value)) {
      if (true_value(value))
        _manifest_list[i]._flags |= QUIET_FAILURE;
    }

    // look for our recursion flag.
    if (ini.get(section_name, "recurse", value)) {
      if (true_value(value))
        _manifest_list[i]._flags |= RECURSIVE_SRC;
    } else {
      // the options here are only appropriate when the target is NOT set to
      // be recursive.

      if (ini.get(section_name, "no_pack", value)) {
        // allow either side to not be required if this is an executable.
        if (true_value(value))
          _manifest_list[i]._flags |= OMIT_PACKING;
      }

      // check if they have specified a source side executable.
      if (ini.get(section_name, "exec_source", value)) {
        if (true_value(value)) {
          _manifest_list[i]._flags |= SOURCE_EXECUTE;
        }
      } else {
        // check if they have specified a target side executable.  this is
        // mutually exclusive with a source side exec.
        if (ini.get(section_name, "exec_target", value)) {
          if (true_value(value))
            _manifest_list[i]._flags |= TARGET_EXECUTE;
        }
      }
    }

    // replace environment variables in the source now...
    _manifest_list[i]._source = parser_bits::substitute_env_vars
        (_manifest_list[i]._source, false);

    // look for wildcards in the source.
    int indy = _manifest_list[i]._source.find("*");

    // see if they specified a keyword on the command line and if this matches.
    // if not we need to abandon this item.
    if (!!_keyword && !_manifest_list[i]._keywords.member(_keyword)) {
      // their keyword choice didn't match what we were told to use.
      noisy_logfile.log(istring("skipping ") + _manifest_list[i]._target
          + " file check; doesn't match keyword \"" + _keyword + "\"");
      continue;
    }

    // we only access the source file here if it's finalized.  we can't do
    // this if the target is supposed to be recursive or if it's got a wildcard
    // pattern in it.
    if (!(_manifest_list[i]._flags & RECURSIVE_SRC) && negative(indy)
        && !(_manifest_list[i]._flags & OMIT_PACKING) ) {
      // access the source file to get its size.
      byte_filer source_file(_manifest_list[i]._source, "rb");
      if (!source_file.good()) {
        LOG(istring("could not access the source file for bundling: ")
            + _manifest_list[i]._source);
        BAIL(69);
      }
      bool okaysize = get_file_size(_manifest_list[i]._source,
          _manifest_list[i]._size, _manifest_list[i]._timestamp);
      if (!okaysize || (_manifest_list[i]._size < 0) ) {
        // this is a failure, but the function complains about it already.
        BAIL(75);
      }
    }
  }

  // patch the manifest list for wildcards and recursive sources.
  for (int i = 0; i < _manifest_list.length(); i++) {
    bundled_chunk curr = _manifest_list[i];

    if (!!_keyword && !curr._keywords.member(_keyword)) {
      // this item's keyword doesn't match the one we were given, so skip it.
      noisy_logfile.log(istring("zapping entry for ") + curr._target
          + "; doesn't match keyword \"" + _keyword + "\"");
      _manifest_list.zap(i, i);
      i--;  // skip back since we eliminated an index.
      continue;
    }

    if (curr._flags & SET_VARIABLE) {
      // we're done working on this.
      continue;
    } else if (curr._flags & RECURSIVE_SRC) {
      // handle a recursive style target.
      int star_indy = curr._source.find("*");
      if (non_negative(star_indy)) {
        // this is currently illegal.  we don't allow recursion + wildcards.
        LOG(istring("illegal combination of recursion and wildcard: ")
            + curr._source);
        BAIL(70);
      }
      // handle the recursive guy.
      int ret = patch_recursive_target(curr._source, curr._target, i);
      if (ret != 0) {
        LOG(istring("failed during packing of recursive source: ")
            + curr._source);
        BAIL(72);
      }
      // take this item out of the picture, since all contents got included.
      _manifest_list.zap(i, i);
      i--;  // skip back since we eliminated an index.
      continue;
    } else if (curr._flags & SOURCE_EXECUTE) {
      // we have massaged the current manifest chunk as much as we can, so now
      // we will execute the source item if that was specified.
      BASE_LOG(istring("launching ") + curr._source);
      if (!!curr._parms)
        BASE_LOG(istring("\tparameters ") + curr._parms);
      BASE_LOG(istring('-', 76));
      u_int kid;
      u_int retval = portable::launch_process(curr._source, curr._parms,
          portable::AWAIT_APP_EXIT, kid);
      if (retval != 0) {
        LOG(istring("failed to launch process, source=") + curr._source
            + ", with parms " + curr._parms);
        if (! (curr._flags & IGNORE_ERRORS) ) {
          BAIL(92);
        }
      }
      BASE_LOG(istring('-', 76));
      if (curr._flags & OMIT_PACKING) {
        // this one shouldn't be included in the package.
        _manifest_list.zap(i, i);
        i--;  // skip back since we eliminated an index.
      }
      continue;
    } else {
      // check for a wildcard.
      int star_indy = curr._source.find("*");
      if (negative(star_indy)) continue;  // simple targets are boring.
      // this does have a wildcard in it.  let's make sure it's in the right
      // place for a wildcard in our scheme.
      int slash_indy = curr._source.find('/', curr._source.end(), true);
      if (star_indy < slash_indy) {
        BASE_LOG(istring("illegal wildcard placement in ") + curr._source);
        BASE_LOG("  (the wildcard must be in the last component of the path)");
        BAIL(71);
      }
      // handle the wildcarded source.
      int ret = patch_wildcard_target(curr._source, curr._target, i);
      if (ret != 0) {
        LOG(istring("failed during packing of wildcarded source: ")
            + curr._source);
        BAIL(73);
      }
      _manifest_list.zap(i, i);
      i--;  // skip back since we eliminated an index.
      continue;
    }
  }

#ifdef DEBUG_BUNDLER
  if (!final_return) {
    // we had a successful run so we can print this stuff out.
    LOG("read the following info from manifest:");
    BASE_LOG("size\tsource\t\t\ttarget");
    for (int i = 0; i < _manifest_list.length(); i++) {
      bundled_chunk &curr = _manifest_list[i];
      BASE_LOG(isprintf("%d\t%s\t\t\t%s", curr._size,
          curr._source.s(), curr._target.s()));
    }
  }
#endif

  return final_return;
}

int bundle_creator::write_stub_and_toc()
{
  FUNCDEF("write_stub_and_toc");

  // define our location to find the unpacking stub program.
//hmmm: make this a command line parameter.
#ifdef __UNIX__
  istring stub_file("$REPOSITORY_DIR/exe/unpacker_stub");
#endif
#ifdef __WIN32__
  istring stub_file("$REPOSITORY_DIR/exe/unpacker_stub.exe");
#endif

  // replace any environment variables on the source side now.
  stub_file = parser_bits::substitute_env_vars(stub_file, false);

  // make sure the stub is accessible.
  byte_filer stubby(stub_file, "rb");
  if (!stubby.good()) {
    LOG(istring("could not access the unpacking stub at: ") + stub_file);
    return 80;
  }
  _stub_size = int(stubby.length());  // get the stub size for later reference.
  byte_array whole_stub;
  stubby.read(whole_stub, _stub_size + 100);
  stubby.close();
  _bundle->write(whole_stub);

  byte_array packed_toc_len;
  basis::obscure_attach(packed_toc_len, _manifest_list.length());
  int ret = _bundle->write(packed_toc_len);
  if (ret < 0) {
    LOG(istring("could not write the TOC length to the bundle: ")
        + _output_file);
    return 81;
  }

  // dump out the manifest list in our defined format.
  for (int i = 0; i < _manifest_list.length(); i++) {
    bundled_chunk &curr = _manifest_list[i];
//LOG(isprintf("flag %d is %d", i, curr._flags));
    byte_array chunk;
    curr.pack(chunk);
    if (_bundle->write(chunk) <= 0) {
      LOG(isprintf("could not write item #%d [%s] to the bundle: ", i,
          curr._source.s())
          + _output_file);
      return 88;
    }
  }

  return 0;
}

int bundle_creator::bundle_sources()
{
  FUNCDEF("bundle_sources");
  // go through all the source files and append them to the bundled output.
  file_logger noisy_logfile(path_configuration::make_logfile_name
      ("bundle_creator_activity.log"));
  for (int i = 0; i < _manifest_list.length(); i++) {
    bundled_chunk &curr = _manifest_list[i];

    if (curr._flags & SET_VARIABLE) {
      // all we need to do is keep this in the manifest.
      noisy_logfile.log(istring("bundling: variable setting ") + curr._target
          + "=" + curr._parms);
      continue;
    } else if (curr._flags & OMIT_PACKING) {
      // this one shouldn't be included in the package.
      continue;
    }

    noisy_logfile.log(istring("bundling: ") + curr._source);
    byte_filer source(curr._source, "rb");
    if (!source.good()) {
      LOG(isprintf("could not read item #%d for the bundle: \"", i)
          + curr._source + "\"");
      return 98;
    }

    byte_array compressed(256 * KILOBYTE);  // expand the buffer to start with.
    byte_array temp;  // temporary read buffer.

    // chew on the file a chunk at a time.  this allows us to easily handle
    // arbitrarily large files rather than reading their entirety into memory.
    int total_written = 0;
    do {
      int ret = source.read(temp, CHUNKING_SIZE);
      if (ret < 0) {
        LOG(isprintf("failed while reading item #%d: ", i) + curr._source);
        return 99;
      } 
      total_written += ret;  // add in what we expect to write.
      // skip compressing if there's no data.
      uLongf destlen = 0;
      bool null_chunk = false;
      if (ret == 0) {
        compressed.reset();
  null_chunk = true;
      } else {
        compressed.reset(int(0.1 * ret) + ret + KILOBYTE);
          // provide some extra space as per zlib instructions.  we're giving it
          // way more than they request.
        destlen = compressed.length();
        // pack the chunks first so we can know sizes needed.
        int comp_ret = compress(compressed.access(), &destlen, temp.observe(),
            temp.length());
        if (comp_ret != Z_OK) {
          LOG(isprintf("failed while compressing item #%d: ", i)
              + curr._source);
          return 99;
        }
        compressed.zap(destlen, compressed.length() - 1);
      }
      byte_array just_sizes;
      basis::obscure_attach(just_sizes, temp.length());
        // add in the real size.
      basis::obscure_attach(just_sizes, int(destlen));
        // add in the packed size.
      ret = _bundle->write(just_sizes);
      if (ret <= 0) {
        LOG(isprintf("failed while writing sizes for item #%d: ", i)
            + curr._source);
        return 93;
      }
      if (!null_chunk) {
        ret = _bundle->write(compressed);
        if (ret <= 0) {
          LOG(isprintf("failed while writing item #%d: ", i) + curr._source);
          return 93;
        } else if (ret != compressed.length()) {
          LOG(isprintf("wrote different size for item #%d (tried %d, "
              "wrote %d): ", i, compressed.length(), ret) + curr._source);
          return 93;
        }
      }
    } while (!source.eof());
//hmmm: very common code to above size writing.
    byte_array just_sizes;
    basis::obscure_attach(just_sizes, -1);
    basis::obscure_attach(just_sizes, -1);
    int ret = _bundle->write(just_sizes);
    if (ret <= 0) {
      LOG(isprintf("failed while writing sentinel of item #%d: ", i)
          + curr._source);
      return 96;
    }
    source.close();
    if (total_written != curr._size) {
      LOG(isprintf("size (%d) disagrees with initial size (%d) for "
          "item #%d: ", total_written, curr._size, i) + curr._source);
    }
  }
  noisy_logfile.log(istring("Bundling run ends at ") + timestamp(false, true));
  noisy_logfile.log(istring('-', 76));

  return 0;
}

int bundle_creator::finalize_file()
{
  _bundle->close();
  return 0;
}

int bundle_creator::write_offset()
{
  FUNCDEF("write_offset");
  byte_filer bun(_output_file, "r+b");  // open the file for updating.

  istring magic_string("muftiloc");  // our sentinel string.
  istring temp_string;  // data from the file.

  while (!bun.eof()) {
    // find the telltale text in the file.
    bool found_it = false;  // we'll set this to true if we see the string.
    int location = 0;  // where the sentinel's end is.
    for (int i = 0; i < magic_string.length(); i++) {
      int ret = bun.read(temp_string, 1);
      if (ret <= 0) break;
      if (temp_string[0] != magic_string[i]) break;  // no match.
      if (i == magic_string.end()) {
        // we found a match to our string!
        found_it = true;
        location = int(bun.tell());
//LOG(isprintf("found the sentinel in the file!  posn=%d", location));
      }
    }
    if (!found_it) continue;  // keep reading.
    bun.seek(location);
    byte_array packed_offset;
    basis::obscure_attach(packed_offset, _stub_size);
//LOG(istring("pattern of len is:\n") + byte_format::text_dump(packed_offset));
    // write the offset into the current position, which should be just after
    // the sentinel's location.
    bun.write(packed_offset);
//LOG(isprintf("wrote manifest offset before posn=%d", bun.tell()));
    break;  // done with looking for that pattern.
  }
  bun.close();  // completely finished now.

  chmod(_output_file.s(), 0766);
    // make sure it's an executable file when we're done with it.

  BASE_LOG(istring("done file bundling at ") + timestamp(false, true));

  return 0;
}


HOOPLE_MAIN(bundle_creator, )

#ifdef __BUILD_STATIC_APPLICATION__
  // static dependencies found by buildor_gen_deps.sh:
  #include <basis/array.cpp>
  #include <basis/byte_array.cpp>
  #include <basis/callstack_tracker.cpp>
  #include <basis/chaos.cpp>
  #include <basis/convert_utf.cpp>
  #include <basis/definitions.cpp>
  #include <basis/earth_time.cpp>
  #include <basis/guards.cpp>
  #include <basis/istring.cpp>
  #include <basis/log_base.cpp>
  #include <basis/memory_checker.cpp>
  #include <basis/mutex.cpp>
  #include <basis/object_base.cpp>
  #include <basis/outcome.cpp>
  #include <basis/packable.cpp>
  #include <basis/portable.cpp>
  #include <basis/sequence.cpp>
  #include <basis/set.cpp>
  #include <basis/utility.cpp>
  #include <basis/version_record.cpp>
  #include <data_struct/amorph.cpp>
  #include <data_struct/bit_vector.cpp>
  #include <data_struct/byte_hasher.cpp>
  #include <data_struct/configurator.cpp>
  #include <data_struct/hash_table.cpp>
  #include <data_struct/pointer_hash.cpp>
  #include <data_struct/stack.cpp>
  #include <data_struct/static_memory_gremlin.cpp>
  #include <data_struct/string_hash.cpp>
  #include <data_struct/string_hasher.cpp>
  #include <data_struct/string_table.cpp>
  #include <data_struct/symbol_table.cpp>
  #include <data_struct/table_configurator.cpp>
  #include <loggers/console_logger.cpp>
  #include <loggers/file_logger.cpp>
  #include <loggers/locked_logger.cpp>
  #include <loggers/null_logger.cpp>
  #include <loggers/program_wide_logger.cpp>
  #include <opsystem/application_base.cpp>
  #include <opsystem/application_shell.cpp>
  #include <opsystem/byte_filer.cpp>
  #include <opsystem/command_line.cpp>
  #include <opsystem/critical_events.cpp>
  #include <opsystem/directory.cpp>
  #include <opsystem/filename.cpp>
  #include <opsystem/filetime.cpp>
  #include <opsystem/ini_config.cpp>
  #include <opsystem/ini_parser.cpp>
  #include <opsystem/path_configuration.cpp>
  #include <opsystem/rendezvous.cpp>
  #include <textual/byte_format.cpp>
  #include <textual/list_parsing.cpp>
  #include <textual/parser_bits.cpp>
  #include <textual/string_manipulation.cpp>
  #include <textual/tokenizer.cpp>
#endif // __BUILD_STATIC_APPLICATION__

---