/*   htmlparse.cc
*
*  Read html an file and generate a tag tree structure.

  The copyright for the following is held by Neil Nelson but may be copied or modified
  by Jure Sah for the purpose of automating or increasing the HTML item extraction
  ability of the MMS or MesonAI related objectives of which Neil Nelson is a part.

*
*/

#include <stdio.h>
#include <fstream>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>

#include <algorithm>
#include <iostream>
#include <string>
#include <functional>
#include <ctype.h>

#include <new.h>

#include "htmlparse.h"

// #define DEBUG_PARSI
// #define DEBUG_TREE
#define JURES_VERSION

// extern ifstream ihtml;
// extern char *res; // debug

#ifdef JURES_VERSION
  int ihtml_eof;
  int shut_down_signal=0;
  void shut_down_handler(int dummy);
#else
  extern int ihtml_eof;
  extern int shut_down_signal;
  extern void shut_down_handler(int dummy);
#endif

extern string get_html_char();

string get_lower(string cap_word, uint length);
void delete_html_tag_tree(html_tag* wrk_tag);
html_tag* get_last_html_tag(html_tag* wrk_tag);
int get_html_file_list(string directory_name);
void out_of_memory();

// html_tag *get_html_tags()
Html_Tag_Classes get_html_tags(Html_Tag_Classes html_tag_classes)
{
  signal(30, shut_down_handler);

  html_tag *cur_tag, *top_tag, *new_tag, *wrk_tag, *prev_tag, *bot_tag, *temp_tag;
  Anchor_Tag *top_anchor_tag, *cur_anchor_tag, *new_anchor_tag;
  top_anchor_tag=0;

  string schar, prev_schar="", sstr="", wrk_str;
  string::size_type wrk_str_len;
  int next_dir=0, end_of_string, tags_created=0, tags_deleted=0; // 1 = down, 2 = right.
  int anchor_tags_created=0;

  sstr = "";
  schar = get_html_char();
  while(schar != "<")
  {
    sstr += schar;
    schar = get_html_char();
  }

  top_tag = cur_tag = new html_tag;
  tags_created+=1;

#ifdef DEBUG_PARSI
  cout << "new html_tag top_tag= " << top_tag << "\n";
#endif

  cur_tag->tagvalue = sstr;
  next_dir=2; // right

  while(!ihtml_eof)
  {
    if (shut_down_signal)
    {
      html_tag_classes.top_tree_tag = 0;
      return html_tag_classes;
    }
#ifdef DEBUG_PARSI
    cout << "top of loop schar= " << schar << "\n";
#endif

    // *****************
    if (schar == "<") // start tag
    {

#ifdef DEBUG_PARSI
    cout << "In schar == < " << "\n";
#endif

      sstr = "";
      while(!ihtml_eof && schar != " " && schar != ">" && sstr != "<!--")
      {
         sstr += schar;
         schar = get_html_char();
      }
      sstr = get_lower(sstr, 0);

      if (sstr == "<!--") // comment area.
      {
        new_tag = new html_tag;
        tags_created+=1;

#ifdef DEBUG_PARSI
    cout << "In sstr == <!--" << " new_tag= " << new_tag << "\n";
#endif

        if (next_dir == 1)
        {
          cur_tag->downtag = new_tag;
          new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
    cout << "In sstr == <!--" << " new_tag= " << new_tag << " down " << "\n";
#endif

        }
        else if (next_dir == 2)
        {
          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
    cout << "In sstr == <!--" << " new_tag= " << new_tag << " right " << "\n";
#endif
        }
        else
        {
          cout << "<h3>error 1 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }
        new_tag->tagname = sstr;
        cur_tag = new_tag;

#ifdef DEBUG_PARSI
    cout << "In sstr == <!-- cur_tag->tagname= " << cur_tag->tagname
         << " cur_tag= " << cur_tag << "\n";
#endif

        wrk_str = sstr = schar;
        while(!ihtml_eof && wrk_str != "-->")
        {
           sstr += get_html_char();
           wrk_str_len = sstr.size();
           if (wrk_str_len > 2) wrk_str = sstr.substr(wrk_str_len-3,3);
        }
        if (ihtml_eof)
        {
          cout << "<h3>error - eof found, expected --> for end of comment.</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }

        sstr = sstr.substr(0,wrk_str_len-3);

        cur_tag->tagvalue = sstr;
        cur_tag->tagend = "-->";
        sstr = ""; // remove?
        next_dir = 2; // right
        // continue at bottom of main loop

#ifdef DEBUG_PARSI
    cout << "In sstr == <!-- cur_tag->tagname= " << cur_tag->tagname
         << " cur_tag->tagvalue= " << cur_tag->tagvalue
         << " cur_tag->tagend= " << cur_tag->tagend
         << "\n";
#endif

      } // end of comment area

      else if (schar == " ")
      {
        new_tag = new html_tag;
        tags_created+=1;

        if (sstr == "<a") // Write anchor tag.
        {
//          anchor_tags_created+=1;
          if (!top_anchor_tag)
            top_anchor_tag = new_anchor_tag = new Anchor_Tag;
          else
          {
            new_anchor_tag = new Anchor_Tag;
            cur_anchor_tag->righttag = new_anchor_tag;
          }
          cur_anchor_tag = new_anchor_tag;
#ifdef DEBUG_PARSI
    cout << "In schar == < then space" << " before html_tag_ref = new_tag new_tag= "
         << new_tag << "\n";
#endif
          cur_anchor_tag->html_tag_ref = new_tag;
        }


#ifdef DEBUG_PARSI
    cout << "In schar == < then space" << " new_tag= " << new_tag << "\n";
#endif

        if (next_dir == 1)
        {
          cur_tag->downtag = new_tag;
          new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
    cout << "In schar == < then space" << " new_tag= " << new_tag << " down " << "\n";
#endif

        }
        else if (next_dir == 2)
        {
          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
    cout << "In schar == < then space" << " new_tag= " << new_tag << " right " << "\n";
#endif
        }
        else
        {
          cout << "<h3>error 1 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }
        new_tag->tagname = sstr;
        cur_tag = new_tag;
        sstr = ""; // remove?
        next_dir = 1; // down

#ifdef DEBUG_PARSI
    cout << "In schar == < then space cur_tag->tagname= " << cur_tag->tagname
         << " cur_tag= " << cur_tag << "\n";
#endif

      }

      else if (schar == ">") // found at end of tag name as in </a>
      {
        sstr += schar;

#ifdef DEBUG_PARSI
    cout << "in schar == < then >" << " sstr= " << sstr << "\n";
#endif

        if (sstr.substr(0, 2) == "</")
        {
          new_tag = new html_tag;
          tags_created+=1;
          new_tag->tagname = sstr;

#ifdef DEBUG_PARSI
    cout << "in schar == < then > in == </ 1" << "\n";
#endif

          wrk_str_len = sstr.size() - 2;
          wrk_str = "<" + sstr.substr(2, wrk_str_len);
          wrk_str_len +=1;

#ifdef DEBUG_PARSI
    cout << "in schar == < then >" << " wrk_str= " << wrk_str
         << " wrk_str_len= " << wrk_str_len <<  " new_tag= " << new_tag <<"\n";
#endif

          bot_tag = wrk_tag = cur_tag;

#ifdef DEBUG_PARSI
    cout << "in schar == < then >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_str= " << wrk_str << " wrk_tag= " << wrk_tag << "\n";
#endif

          while(wrk_tag > 0 && (wrk_tag->tagname + wrk_tag->tagend) != wrk_str)
          {
            prev_tag = wrk_tag;
            if (wrk_tag->lefttag > 0)
            {
               wrk_tag = wrk_tag->lefttag;

#ifdef DEBUG_PARSI
    if (wrk_tag > 0)
    cout << "in schar == < then >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_str= " << wrk_str << " wrk_tag= " << wrk_tag << " left" << "\n";
#endif

            }
            else
            {
              wrk_tag = wrk_tag->uptag;

#ifdef DEBUG_PARSI
    if (wrk_tag > 0)
    cout << "in schar == < then >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_str= " << wrk_str << " wrk_tag= " << wrk_tag << " up" << "\n";
#endif

            }

            if (prev_tag == wrk_tag)
            {
              cout << "<h3>in schar == < then > prev_tag == wrk_tag= " << prev_tag << "</h3>\n";
              cout.flush();
              html_tag_classes.top_tree_tag = 0;
              return html_tag_classes;
            }

          }
          if (wrk_tag == 0)
          {
            if (wrk_str == "<font>")
            {
              cur_tag = bot_tag; // Fix this later.
              delete new_tag;
              tags_deleted+=1;

              schar = get_html_char();
              while(!ihtml_eof && schar == " ") schar = get_html_char(); // bottom read
              continue;
            }
            else
            { 
              cur_tag = bot_tag; // Fix this later.
              delete new_tag;
              tags_deleted+=1;

              schar = get_html_char();
              while(!ihtml_eof && schar == " ") schar = get_html_char(); // bottom read
              continue;

              cout << "<h3>error - top tag? left or up tag to be non zero.  sstr= " << sstr
                     << "</h3>\n";
              cout.flush();
              html_tag_classes.top_tree_tag = 0;
              return html_tag_classes;
            }
          }

          if (wrk_tag->righttag > 0)
          {
            temp_tag = wrk_tag->righttag;

#ifdef DEBUG_PARSI
    cout << "in schar == < then > before (wrk_tag->righttag)->tagname"
         << " sstr= " << sstr << " temp_tag= " << temp_tag
         << " wrk_tag= " << wrk_tag << "\n";
#endif

            if ((temp_tag->tagname + temp_tag->tagend) == sstr)
            {
              cur_tag = bot_tag; // Fix this later.
              delete new_tag;
              tags_deleted+=1;

              schar = get_html_char();
              while(!ihtml_eof && schar == " ") schar = get_html_char(); // bottom read
              continue;

              cout << "<h3>error - end of tag already found.  sstr= " << sstr << "</h3>\n";
              cout.flush();
              html_tag_classes.top_tree_tag = 0;
              return html_tag_classes;
            }
          }

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 100" << " sstr= " << sstr << " down/right" << "\n";
#endif

          new_tag->downtag = wrk_tag->righttag;
          wrk_tag->righttag = new_tag;
          new_tag->lefttag = wrk_tag;

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 100" << " sstr= " << sstr << " down/right"
         << " new_tag->downtag= " << new_tag->downtag
         << " wrk_tag->righttag= " << wrk_tag->righttag
         << " new_tag->lefttag= " << new_tag->lefttag << "\n";
#endif

          if (new_tag->downtag > 0)
          {
            wrk_tag = new_tag->downtag;

#ifdef DEBUG_PARSI
    cout << "if (new_tag->downtag > 0) - before wrk_tag->uptag = new_tag" << "\n";
#endif

            wrk_tag->uptag = new_tag;
            wrk_tag->lefttag = 0;
          }

#ifdef DEBUG_PARSI
    cout << "error - before new_tag->tagname = sstr" << "\n";
#endif

          new_tag->tagname = sstr;

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 100.5" << " sstr= " << sstr << " down/right"
         << " new wrk_tag->uptag= " << wrk_tag->uptag
         << " new_tag->tagname= " << new_tag->tagname << "\n";
#endif

        }
        else if (sstr == "<script")
        {
          cout << "<h3>error - did not expect <script>" << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }
        else if (sstr.substr(0,1) == "<")
        {
          new_tag = new html_tag;
          tags_created+=1;

          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->tagname = sstr;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 101" << " sstr= " << sstr << " right"
         << " cur_tag->righttag= " << cur_tag->righttag
         << " cur_tag->lefttag= " << cur_tag->lefttag
         << " new_tag->lefttag= " << new_tag->lefttag
         << " new_tag->uptag= " << new_tag->uptag << "\n";
#endif

        }
        else
        {
          cout << "<h3>error - expected complete tag as in <a>.  sstr= " << sstr << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 102" << " sstr= " << sstr << "\n";
#endif

        sstr = ""; // remove?
        cur_tag = new_tag;
        next_dir=2; // right

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 103" << " sstr= " << sstr
         << " cur_tag->tagname= " << cur_tag->tagname
         << " cur_tag= " << cur_tag << "\n";
#endif

        // continue at bottom of main loop
      }
      else if (ihtml_eof)
      {
        cout << "<h3>error - did not expect html_eof.  sstr= " << sstr << "</h3>\n";
        cout.flush();
        html_tag_classes.top_tree_tag = 0;
        return html_tag_classes;
      }
      else
      {
        cout << "<h3>error - expected either space or >.  schar= " << schar
             << " sstr= " << sstr << "</h3>\n";
        cout.flush();
        html_tag_classes.top_tree_tag = 0;
        return html_tag_classes;
      }
      // continue at bottom of main loop

#ifdef DEBUG_PARSI
    cout << "in schar == < then > 104  continue at bottom of loop \n";
#endif

    }


    // *****************
    else if (schar == ">") // end html_tag
    {

#ifdef DEBUG_PARSI
    cout << "In schar == >" << "\n";
#endif

      wrk_tag = cur_tag;

#ifdef DEBUG_PARSI
    if (wrk_tag > 0)
    cout << "in schar == >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_tag= " << wrk_tag << "\n";
#endif

      while(wrk_tag > 0 && wrk_tag->tagname.substr(0,1) != "<")
      {
        if (wrk_tag->lefttag > 0)
        {
          wrk_tag = wrk_tag->lefttag;

#ifdef DEBUG_PARSI
    if (wrk_tag > 0)
    cout << "in schar == >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_tag= " << wrk_tag << " left" << "\n";
#endif

        }
        else
        {
          wrk_tag = wrk_tag->uptag;

#ifdef DEBUG_PARSI
    if (wrk_tag > 0)
    cout << "in schar == >" << " wrk_tag->tagname= " << wrk_tag->tagname
         << " wrk_tag= " << wrk_tag << " up" << "\n";
#endif

        }

      }
      if (wrk_tag == 0)
      {
        cout << "<h3>error - last left tag and match to > not found last left wrk_tag->tagname= "
             << wrk_tag->tagname << "</h3>\n";
        cout.flush();
        html_tag_classes.top_tree_tag = 0;
        return html_tag_classes;
      }
      wrk_tag->tagend = ">";
      cur_tag = wrk_tag;
      sstr = ""; // remove?
      next_dir=2; // right
      // continue at bottom of main loop

#ifdef DEBUG_PARSI
      cout << "> found wrk_tag->tagname= " << wrk_tag->tagname
           << " wrk_tag= " << wrk_tag << "\n";
#endif

    }


    // *****************
    else if (schar == "\"") // get quoted string.
    {

#ifdef DEBUG_PARSI
    cout << "In schar == double quote" << "\n";
#endif

      if (cur_tag->tagname.substr(cur_tag->tagname.size()-1,1) == "=")
      {
        sstr = schar;
        schar = get_html_char();
        while(!ihtml_eof && schar != "\"")
        {
          sstr += schar;
          schar = get_html_char();
        }
        if (ihtml_eof)
        {
          cout << "<h3>error - found unexpected ihtml_eof. sstr[0:20]= " << sstr.substr(0,20)
               << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }
        cur_tag->tagvalue = sstr + "\"";
        sstr = ""; // remove?
        next_dir=2; // right
       // continue at bottom of main loop
      }
      else
      {

#ifdef DEBUG_PARSI
      cout << "in double quote else remainder schar= " << schar << "\n";
#endif

        sstr = "";
        while(!ihtml_eof && schar != "<") // && !end_of_string 
        {
           sstr += schar;
           schar = get_html_char();
        }
        if (ihtml_eof) continue; // Continue at top and exit loop.

        new_tag = new html_tag;
        tags_created+=1;
        new_tag->tagvalue = sstr;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue << "\n";
#endif

        if (next_dir == 1)
        {
          cur_tag->downtag = new_tag;
          new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue
           << " down " << "\n";
#endif

        }
        else if (next_dir == 2)
        {
          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue
           << " eight " << "\n";
#endif

        }
        else
        {
          cout << "<h3>error 3 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }

        cur_tag = new_tag;
        sstr = ""; // remove?
        next_dir = 2; // right

#ifdef DEBUG_PARSI
      cout << "end of double quote else sequence" << " cur_tag= " << cur_tag
           << " cur_tag->tagvalue = " << cur_tag->tagvalue << "\n";
#endif

        if (schar == "<") continue; // Continue at top without bottom read.
        // Continue at bottom read.

      }
      // continue at bottom of main loop

#ifdef DEBUG_PARSI
      cout << "double quote found cur_tag->tagname= " << cur_tag->tagname
           << " cur_tag->tagvalue= " << cur_tag->tagvalue << "\n";
#endif

    }


    // *****************
    else if (schar == "'") // get quoted string.
    {

#ifdef DEBUG_PARSI
    cout << "In schar == single quote" << "\n";
#endif

      if (cur_tag->tagname.substr(cur_tag->tagname.size()-1) == "=")
      {
        sstr = schar;
        schar = get_html_char();
        while(!ihtml_eof && schar != "'")
        {
          sstr += schar;
          schar = get_html_char();
        }
        if (ihtml_eof)
        {
          cout << "<h3>error - found unexpected ihtml_eof. sstr[0:20]= " << sstr.substr(0,20)
               << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }
        cur_tag->tagvalue = sstr + "'";
        sstr = ""; // remove?
        next_dir=2; // right
        // continue at bottom of main loop
      }
      else
      {

#ifdef DEBUG_PARSI
      cout << "in single quote else remainder schar= " << schar << "\n";
#endif

        sstr = "";
        while(!ihtml_eof && schar != "<") // && !end_of_string 
        {
           sstr += schar;
           schar = get_html_char();
        }
        if (ihtml_eof) continue; // Continue at top and exit loop.

        new_tag = new html_tag;
        tags_created+=1;
        new_tag->tagvalue = sstr;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue << "\n";
#endif

        if (next_dir == 1)
        {
          cur_tag->downtag = new_tag;
          new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue
           << " down " << "\n";
#endif

        }
        else if (next_dir == 2)
        {
          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
      cout << "after while loop new_tag->tagvalue= " << new_tag->tagvalue
           << " eight " << "\n";
#endif

        }
        else
        {
          cout << "<h3>error 3 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }

        cur_tag = new_tag;
        sstr = ""; // remove?
        next_dir = 2; // right

#ifdef DEBUG_PARSI
      cout << "end of single quote else sequence" << " cur_tag= " << cur_tag
           << " cur_tag->tagvalue = " << cur_tag->tagvalue << "\n";
#endif

        if (schar == "<") continue; // Continue at top without bottom read.
        // Continue at bottom read.
      }
      // continue at bottom of main loop

#ifdef DEBUG_PARSI
      cout << "single quote found cur_tag->tagname= " << cur_tag->tagname
           << " cur_tag->tagvalue= " << cur_tag->tagvalue << "\n";
#endif

    }


    // *****************
    else if (schar == "=") // equal type 
    { // On second = sign, bypass and go to bottom read
    }


    // *****************
    else
    {

#ifdef DEBUG_PARSI
      cout << "in else else remainder schar= " << schar << "\n";
#endif
      if (     (cur_tag->tagname.size() > 0
             && cur_tag->tagname.substr(cur_tag->tagname.size()-1,1) == ">")
          || cur_tag->tagend == ">")
      {

#ifdef DEBUG_PARSI
      cout << "in else else remainder after first if schar= " << schar << "\n";
#endif

        sstr = "";
        while(!ihtml_eof && schar != "<") // && !end_of_string 
        {
           sstr += schar;
           schar = get_html_char();
        }
        if (ihtml_eof) continue; // Continue at top and exit loop.

        new_tag = new html_tag;
        tags_created+=1;
        new_tag->tagvalue = sstr;

#ifdef DEBUG_PARSI
      cout << "after while loop 12 new_tag->tagvalue= " << new_tag->tagvalue << "\n";
#endif

        if (next_dir == 1)
        {
          cur_tag->downtag = new_tag;
          new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
      cout << "after while loop 12 new_tag->tagvalue= " << new_tag->tagvalue
           << " down " << "\n";
#endif

        }
        else if (next_dir == 2)
        {
          cur_tag->righttag = new_tag;
          new_tag->lefttag = cur_tag;
          new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
      cout << "after while loop 12 new_tag->tagvalue= " << new_tag->tagvalue
           << " eight " << "\n";
#endif

        }
        else
        {
          cout << "<h3>error 3 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
          cout.flush();
          html_tag_classes.top_tree_tag = 0;
          return html_tag_classes;
        }

        cur_tag = new_tag;
        sstr = ""; // remove?
        next_dir = 2; // right

#ifdef DEBUG_PARSI
      cout << "end of > sequence 13" << " cur_tag= " << cur_tag
           << " cur_tag->tagvalue = " << cur_tag->tagvalue << "\n";
#endif

        if (schar == "<") continue; // Continue at top without bottom read.
        // Continue at bottom read.
      }

      else if (cur_tag->tagname.substr(0,1) == "<") // Following of start-of-tag type.
      {
        while(!ihtml_eof && schar != ">" && schar != "=")
        {
          end_of_string=0;
          sstr = "";

          while(!ihtml_eof && !end_of_string && schar != ">")
          {
             sstr += schar;
             schar = get_html_char();
             if (schar == " ")
             {
               while(!ihtml_eof && schar == " ") schar = get_html_char();
               end_of_string=1;
             }
             if (schar == "=")
             {
               sstr += "=";
               end_of_string=1;
             }
          }
          if (ihtml_eof) break;

          new_tag = new html_tag;
          tags_created+=1;

          if (schar == "=") new_tag->tagname = sstr;
          else new_tag->tagvalue = sstr;

#ifdef DEBUG_PARSI
      cout << "after while loop 10 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << "\n";
#endif

          if (next_dir == 1)
          {
            cur_tag->downtag = new_tag;
            new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
      cout << "after while loop 10 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << " down " << "\n";
#endif

          }
          else if (next_dir == 2)
          {
            cur_tag->righttag = new_tag;
            new_tag->lefttag = cur_tag;
            new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
      cout << "after while loop 10 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << " right " << "\n";
#endif

          }
          else
          {
            cout << "<h3>error 2 - expected next_dir= 1 or 2, next_dir= " << next_dir << "</h3>\n";
            cout.flush();
            html_tag_classes.top_tree_tag = 0;
            return html_tag_classes;
          }

          cur_tag = new_tag;
          sstr = ""; // remove?
          next_dir = 2; // right

#ifdef DEBUG_PARSI
      cout << "end of < sequence 11" << " cur_tag->tagname= " << cur_tag->tagname
           << " cur_tag->tagvalue= " << cur_tag->tagvalue
           << " cur_tag= " << cur_tag << "\n";
#endif

        }
        if (schar == ">") continue; // Char already obtained. No bottom read required.
        // Continue at bottom read.

#ifdef DEBUG_PARSI
      cout << "end of < sequence 11" << "\n";
#endif

      }

      else 
      {

#ifdef DEBUG_PARSI
        cout << "in else else remainder schar= " << schar << "\n";
#endif

        while(!ihtml_eof && schar != ">" && schar != "=")
        {
          end_of_string=0;
          sstr = "";

#ifdef DEBUG_PARSI
          cout << "in else remainder while 1 schar= " << schar << " sstr= " << sstr << "\n";
#endif

          while(!ihtml_eof && !end_of_string && schar != ">")
          {
             sstr += schar;
             schar = get_html_char();
             if (schar == " ")
             {
               while(!ihtml_eof && schar == " ") schar = get_html_char();
               end_of_string=1;
             }
             if (schar == "=")
             {
               sstr += "=";
               end_of_string=1;
             }

#ifdef DEBUG_PARSI
              cout << "in else remainder end while 2 schar= " << schar
                   << " sstr= " << sstr << "\n";
#endif
          }

#ifdef DEBUG_PARSI
          cout << "in else remainder after while 2 schar= " << schar
                   << " sstr= " << sstr << "\n";
#endif

          if (   schar != "="
              && cur_tag->tagname.size() > 0
              && cur_tag->tagname.substr(cur_tag->tagname.size()-1,1) == "="
              && cur_tag->tagvalue == "")
          {

#ifdef DEBUG_PARSI
          cout << "value without quotes sstr= " << sstr << " schar= " << schar << "\n";
#endif

            cur_tag->tagvalue = sstr;
            sstr = ""; // remove?
            next_dir = 2; // right
            // Following two instructions not necessary because of outer while loop.
            // if (schar == ">") continue; // Bypass bottom read. 
            // Else continue at bottom read.
          }

          else
          {
            new_tag = new html_tag;
            tags_created+=1;

            if (schar == "=") new_tag->tagname = sstr;
            else new_tag->tagvalue = sstr;

#ifdef DEBUG_PARSI
      cout << "after while loop 13 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << "\n";
#endif

            if (next_dir == 1)
            {
              cur_tag->downtag = new_tag;
              new_tag->uptag = cur_tag;

#ifdef DEBUG_PARSI
      cout << "after while loop 13 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << " down" <<  "\n";
#endif

            }
            else if (next_dir == 2)
            {
              cur_tag->righttag = new_tag;
              new_tag->lefttag = cur_tag;
              new_tag->uptag = cur_tag->uptag;

#ifdef DEBUG_PARSI
      cout << "after while loop 13 new_tag->tagname= " << new_tag->tagname
           << " new_tag->tagvalue= " << new_tag->tagvalue << " right" <<  "\n";
#endif

            }
            else
            {
              cout << "<h3>error 2 - expected next_dir= 1 or 2, next_dir= "
                   << next_dir << "</h3>\n";
              cout.flush();
              html_tag_classes.top_tree_tag = 0;
              return html_tag_classes;
            }

            cur_tag = new_tag;
            sstr = ""; // remove?
            next_dir = 2; // right

#ifdef DEBUG_PARSI
      cout << "end of < sequence 14" << " cur_tag->tagname= " << cur_tag->tagname
           << " cur_tag->tagvalue= " << cur_tag->tagvalue
           << " cur_tag= " << cur_tag << "\n";
#endif

          }
        }
        if (schar == ">") continue; // Char already obtained. No bottom read required.
        // Continue at bottom read.

#ifdef DEBUG_PARSI
      cout << "end of < sequence 15" << "\n";
#endif

      }
    }

    schar = get_html_char();
    while(!ihtml_eof && schar == " ") schar = get_html_char(); // bottom read
  }
#ifdef DEBUG_PARSI
  cout << "return top_tag tagname= " << top_tag->tagname << " tagvalue= "
       << top_tag->tagvalue.substr(0,wrk_str_len)
       << " top_tag= " << top_tag << "\n";
#endif

//  cout << "get_tags" << "\n";
//  cout << "tags_created= " << tags_created << "\n";
//  cout << "tags_deleted= " << tags_deleted << "\n";

  html_tag_classes.top_tree_tag = top_tag;
  html_tag_classes.top_anchor_tag = top_anchor_tag;
  return html_tag_classes;
}


string get_lower(string cap_word, uint length)
{
  string::size_type chr_pos, word_len = cap_word.size();

  if (length > 0 && length < word_len)
    for (chr_pos=0;chr_pos<length;chr_pos++)
        cap_word.at(chr_pos) = tolower(cap_word.at(chr_pos));
  else
    for (chr_pos=0;chr_pos<word_len;chr_pos++)
        cap_word.at(chr_pos) = tolower(cap_word.at(chr_pos));

  return cap_word;
}


void delete_html_tag_tree(html_tag* wrk_tag)
{
  html_tag* dlt_tag;
  int first_tag=0, tags_deleted=0;
  string::size_type wrk_str_len; // debug

#ifdef DEBUG_PARSI
  cout << "delete_tag_tree wrk_tag= " << wrk_tag << "\n";
#endif

  first_tag = 0;
  while(!first_tag)
  {
    dlt_tag = get_last_html_tag(wrk_tag);

#ifdef DEBUG_PARSI
    wrk_str_len = dlt_tag->tagvalue.size();
    if (wrk_str_len > 20) wrk_str_len = 20;
    cout << "delete_tag_tree wrk_tag= " << wrk_tag
         << " dlt_tag= " << dlt_tag
         << " L= " << dlt_tag->lefttag
         << " U= " << dlt_tag->uptag
         << " R= " << dlt_tag->righttag
         << " D= " << dlt_tag->downtag
         << " " << dlt_tag->tagname << " " << dlt_tag->tagvalue.substr(0,wrk_str_len)
         << "\n"; 
#endif

    // Get tag next to last.
    if      (dlt_tag->lefttag > 0)
    {
       wrk_tag = dlt_tag->lefttag;
       wrk_tag->righttag=0;
    }
    else if (dlt_tag->uptag > 0)
    {
      wrk_tag = dlt_tag->uptag;
      wrk_tag->downtag=0;
    }
    else first_tag=1;

    tags_deleted +=1;

#ifdef DEBUG_PARSI
    cout << "\ndeleted wrk_tag= " << wrk_tag << " dlt_tag= " << dlt_tag
         << dlt_tag->tagname << " " << dlt_tag->tagvalue << "\n\n";
#endif

    delete dlt_tag;
  }

//  cout << "\ndelete_tag_tree\n";
//  cout << "tags_deleted= " << tags_deleted << "\n";
}


html_tag* get_last_html_tag(html_tag* wrk_tag) // Used in delete_tag_tree
{
  int last_tag;
  string::size_type wrk_str_len; // debug

  if (wrk_tag <= 0)
  {
    cout << "<h3>error - get_last_tag - invalid tag#= " << wrk_tag << "</h3>\n";
    cout.flush();
    return 0;
  }

  last_tag=0;
  while(!last_tag)
  {

#ifdef DEBUG_PARSI
    wrk_str_len = wrk_tag->tagvalue.size();
    if (wrk_str_len > 20) wrk_str_len = 20;
    cout << "get_last_tag wrk_tag= " << wrk_tag
         << " U= " << wrk_tag->uptag
         << " R= " << wrk_tag->righttag
         << " D= " << wrk_tag->downtag
         << " L= " << wrk_tag->lefttag
         << " " << wrk_tag->tagname << " " << wrk_tag->tagvalue.substr(0,wrk_str_len)
         << "\n";
#endif

    if      (wrk_tag->downtag > 0)  wrk_tag = wrk_tag->downtag;
    else if (wrk_tag->righttag > 0) wrk_tag = wrk_tag->righttag;
    else last_tag = 1;
  }
  return wrk_tag;
}

void out_of_memory()
{
  cout << "<h3>Memory exhausted. Program terminates.</h3>\n";
  cout.flush();
//  exit(1);
}

int write_html_tree(string file_name, html_tag *top_tag)
{
  int last_tag=0, next_branch, temp_i=0, end_null=0, prslt, tags_printed=0, tags_deleted=0;
  int stack_tags_created=0, stack_tags_deleted=0;
  string out_string, in_type="", in_name, in_value, indent="";
  html_tag *wrk_tag;
  stack_html_tag *cur_stack_tag, *prev_stack_tag;
  string::size_type wrk_str_len;

  ofstream tree_out;
  tree_out.open(file_name.c_str());

  if (!tree_out)
  {
    cout << "could not open tree_out file_name= " << file_name << "\n";
    return 0;
  }
            
  cur_stack_tag = new stack_html_tag;
  stack_tags_created+=1;
  cur_stack_tag->st_html_tag = wrk_tag = top_tag;


#ifdef DEBUG_TREE
  wrk_str_len = top_tag->tagvalue.size();
  if (wrk_str_len > 20) wrk_str_len = 40;
  cout << "top tag tagname= " << top_tag->tagname << " tagvalue= "
       << top_tag->tagvalue.substr(0,wrk_str_len)
       << " top_tag= " << top_tag << "\n";

      cout << "top_tag= " << wrk_tag << " top_tag->tagname= " << top_tag->tagname
           << " top_tag->righttag= " << top_tag->righttag
           << " top_tag->downtag= " << top_tag->downtag << "\n";
#endif

  tree_out << "top tag " << top_tag->tagname << top_tag->tagvalue.substr(0,wrk_str_len) << "\n";

#ifdef DEBUG_TREE
   cout << "after first tree_out \n";
#endif

  tags_printed+=1;

  while(!last_tag)
  {

#ifdef DEBUG_TREE
   cout << "top of while \n";
#endif

    if (wrk_tag->downtag > 0)
    {

#ifdef DEBUG_TREE
   cout << "1 after first if \n";
#endif

      indent+=".";
      wrk_tag = wrk_tag->downtag;

#ifdef DEBUG_TREE
   cout << "2 after first if \n";
#endif


      prev_stack_tag = cur_stack_tag; // push stack.
      cur_stack_tag = new stack_html_tag;
      stack_tags_created+=1;

#ifdef DEBUG_TREE
   cout << "3 after first if \n";
#endif

      cur_stack_tag->prev_stack_html_tag = prev_stack_tag;
      cur_stack_tag->st_html_tag = wrk_tag;

      wrk_str_len = wrk_tag->tagvalue.size();
      if (wrk_str_len > 80) wrk_str_len = 80;

#ifdef DEBUG_TREE
      cout << "1 wrk_tag= " << wrk_tag
           << " U= " << wrk_tag->uptag
           << " R= " << wrk_tag->righttag
           << " D= " << wrk_tag->downtag
           << " L= " << wrk_tag->lefttag
           << " wrk_tag->tagname= " << wrk_tag->tagname
           << " wrk_tag->value= " << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";
#endif

#ifdef DEBUG_TREE
      cout << wrk_tag << indent << " "
#else
      tree_out << indent << " "
#endif
           << wrk_tag->tagname << " "
           << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";


      tags_printed+=1;
    }
    else if (wrk_tag->righttag > 0)
    {

#ifdef DEBUG_TREE
   cout << "1 after else if \n";
#endif

      cur_stack_tag->st_html_tag = wrk_tag = wrk_tag->righttag;

      wrk_str_len = wrk_tag->tagvalue.size();
      if (wrk_str_len > 80) wrk_str_len = 80;

#ifdef DEBUG_TREE
      cout << "2 wrk_tag= " << wrk_tag
           << " U= " << wrk_tag->uptag
           << " R= " << wrk_tag->righttag
           << " D= " << wrk_tag->downtag
           << " L= " << wrk_tag->lefttag
           << " wrk_tag->tagname= " << wrk_tag->tagname
           << " wrk_tag->value= " << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";
#endif

#ifdef DEBUG_TREE
      cout << wrk_tag << indent
#else
      tree_out << indent << " "
#endif
           << wrk_tag->tagname << " "
           << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";


      tags_printed+=1;
    }
    else
    {

#ifdef DEBUG_TREE
   cout << "1 after else \n";
#endif

      if (cur_stack_tag->prev_stack_html_tag == 0) last_tag = 1;
      else
      {
        // pop stack.
        while(wrk_tag->righttag == 0 && cur_stack_tag->prev_stack_html_tag != 0)
        {
          prev_stack_tag = cur_stack_tag;
          cur_stack_tag = cur_stack_tag->prev_stack_html_tag;
          wrk_tag = cur_stack_tag->st_html_tag;
          delete prev_stack_tag;
          stack_tags_deleted+=1;

          indent = indent.substr(0,indent.size()-1);
        }
        if (wrk_tag->righttag == 0) last_tag = 1;
        else
        {
          wrk_tag = wrk_tag->righttag;
          prev_stack_tag = cur_stack_tag->prev_stack_html_tag;
          delete cur_stack_tag;
          stack_tags_deleted+=1;

          cur_stack_tag = new stack_html_tag;
          stack_tags_created+=1;

          cur_stack_tag->prev_stack_html_tag = prev_stack_tag;
          cur_stack_tag->st_html_tag = wrk_tag;

          wrk_str_len = wrk_tag->tagvalue.size();
          if (wrk_str_len > 80) wrk_str_len = 80;

#ifdef DEBUG_TREE
      cout << "3 wrk_tag= " << wrk_tag
           << " U= " << wrk_tag->uptag
           << " R= " << wrk_tag->righttag
           << " D= " << wrk_tag->downtag
           << " L= " << wrk_tag->lefttag
           << " wrk_tag->tagname= " << wrk_tag->tagname
           << " wrk_tag->value= " << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";

#endif

#ifdef DEBUG_TREE
          cout << wrk_tag << indent
#else
          tree_out << indent << " "
#endif
               << wrk_tag->tagname << " "
               << wrk_tag->tagvalue.substr(0,wrk_str_len) << "\n";

          tags_printed+=1;
        }
      }
    }
  }

  delete cur_stack_tag;
  stack_tags_deleted+=1;

  tree_out << "tags_printed      = " << tags_printed << "\n";
  tree_out << "tags_deleted      = " << tags_deleted << "\n";
  tree_out << "stack_tags_created= " << stack_tags_created << "\n";
  tree_out << "stack_tags_deleted= " << stack_tags_deleted << "\n";

  tree_out.close();
  return 1;
}

void delete_anchor_tags(Anchor_Tag* wrk_tag)
{
  
  Anchor_Tag *sav_tag;
  while(wrk_tag > 0)
  {
    sav_tag = wrk_tag->righttag;
    delete wrk_tag;
    wrk_tag = sav_tag;
  }
}

html_tag *get_next_html_after_loop_tag(html_tag *wrk_tag)
{
  if (wrk_tag->righttag > 0)
  {
    wrk_tag = wrk_tag->righttag;
  }
  else
  {
//  cout << "in get_next_group_tag else wrk_tag->righttag= " << wrk_tag->righttag << "\n";
    while(wrk_tag && wrk_tag->righttag == 0)
    {
      while(wrk_tag->lefttag > 0)
      {
//  cout << "in get_next_group_tag else while wrk_tag->lefttag= " << wrk_tag->lefttag << "\n";
        wrk_tag = wrk_tag->lefttag;
      }
//  cout << "in get_next_group_tag else while wrk_tag->uptag= " << wrk_tag->uptag << "\n";
      wrk_tag = wrk_tag->uptag;
    }
    if (wrk_tag) wrk_tag = wrk_tag->righttag;
  }
//  cout << "in get_next_group_tag return wrk_tag= " << wrk_tag << "\n";
  return wrk_tag;
}

html_tag *get_next_html_group_tag(html_tag *wrk_tag)
{
//  int last_tag=0;

//  cout << "in get_next_html_group_tag wrk_tag= " << wrk_tag << "\n";
//  cout << "in get_next_html_group_tag wrk_tag= " << wrk_tag << " name= " << wrk_tag->tagname << "\n";

  if (wrk_tag->downtag > 0)
  {
    wrk_tag = wrk_tag->downtag;
  }
  else if (wrk_tag->righttag > 0)
  {
    wrk_tag = wrk_tag->righttag;
  }
  else
  {
//  cout << "in get_next_group_tag else wrk_tag->righttag= " << wrk_tag->righttag << "\n";
    while(wrk_tag && wrk_tag->righttag == 0)
    {
      while(wrk_tag->lefttag > 0)
      {
//  cout << "in get_next_group_tag else while wrk_tag->lefttag= " << wrk_tag->lefttag << "\n";
        wrk_tag = wrk_tag->lefttag;
      }
//  cout << "in get_next_group_tag else while wrk_tag->uptag= " << wrk_tag->uptag << "\n";
      wrk_tag = wrk_tag->uptag;
    }
    if (wrk_tag) wrk_tag = wrk_tag->righttag;
  }
//  cout << "in get_next_group_tag return wrk_tag= " << wrk_tag << "\n";
  return wrk_tag;
}

int write_anchor_tags(string anchor_file_name, Anchor_Tag *wrk_tag)
{
/*
  cout << "in write_anchor_tags<br>\n";
  cout << "in write_anchor_tags anchor_file_name= "
       << anchor_file_name << "<br>\n";
  cout << "in write_anchor_tags wrk_tag= "
       << wrk_tag << "<br>\n";
  cout << "in write_anchor_tags wrk_tag->html_tag_ref= "
       << wrk_tag->html_tag_ref << "<br>\n";
  cout.flush();
*/
  ofstream anchor_out;
  anchor_out.open(anchor_file_name.c_str());

  if (!anchor_out)
  {
    cout << "could not open tree_out anchor_file_name= " << anchor_file_name << "\n";
    return 0;
  }

  html_tag *end_tag, *next_tag;
  while(wrk_tag > 0)
  {
    next_tag = wrk_tag->html_tag_ref;
//      anchor_out << next_tag->tagname << "  " << next_tag->tagvalue << "\n";

    end_tag = get_next_html_after_loop_tag(next_tag);
    while(next_tag != end_tag)
    {
      anchor_out << next_tag->tagname << "  " << next_tag->tagvalue << "\n";
      next_tag = get_next_html_group_tag(next_tag);
    }
    anchor_out << "\n";
    next_tag = end_tag;
    end_tag = get_next_html_after_loop_tag(end_tag);
    while(next_tag != end_tag)
    {
      anchor_out << next_tag->tagname << "  " << next_tag->tagvalue << "\n";
      next_tag = get_next_html_group_tag(next_tag);
    }
    anchor_out << "\n";
    anchor_out << "\n";

    wrk_tag = wrk_tag->righttag;
  }

  anchor_out.close();
  return 1;
}

void shut_down_handler(int dummy)
{
  shut_down_signal=1;
}