weHtmlEntity.cpp

Go to the documentation of this file.

/*
    webEngine is the HTML processing library
    Copyright (C) 2009 Andrew Abramov aabramov@ptsecurity.ru

    This file is part of webEngine

    webEngineis free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    webEngineis distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with webEngine.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <time.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/compare.hpp>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/regex.hpp>
#include "weHtmlEntity.h"
#include "weHelper.h"

using namespace boost;
using namespace boost::algorithm;

#ifndef __DOXYGEN__
//WeCompareMode WeHtmlEntity::compareMode = WeCompatreStrict;
#endif //__DOXYGEN__

static bool WeParseNeedToBreakTag(const string& tagName, const string& nextTag)
{
    bool retval = false;

    if (iequals(tagName, "br")) {
        return true;
    }
    if (iequals(tagName, "img")) {
        return true;
    }
    if (iequals(tagName, "base")) {
        return true;
    }
    if (iequals(tagName, "link")) {
        return true;
    }
    if (iequals(tagName, "input")) {
        return true;
    }
    if (iequals(tagName, "li")) {
        if (iequals(nextTag, "li")) {
            return true;
        }
    }
    if (iequals(tagName, "tr")) {
        if (iequals(nextTag, "tr")) {
            return true;
        }
//         if (iequals(nextTag, "table")) {
//             return true;
//         }
    }
    if (iequals(tagName, "td")) {
        if (iequals(nextTag, "td")) {
            return true;
        }
        if (iequals(nextTag, "tr")) {
            return true;
        }
    }

    return retval;
}

WeHtmlEntity::WeHtmlEntity(iweEntity* prnt /*= NULL*/) // :
//    chldList(), attributes()
{
    chldList.resize(0);
    attributes.clear();
    parent = prnt;
    entityName = "";
    GenerateId();
}

WeHtmlEntity::WeHtmlEntity(WeHtmlEntity &entity)
{
    chldList = entity.chldList;
    attributes = entity.attributes;
    parent = entity.parent;
    entityName = entity.entityName;
    GenerateId();
}

WeHtmlEntity::~WeHtmlEntity(void)
{
    ClearAttr();
    ClearChildren();
}

const string& WeHtmlEntity::InnerText(void)
{
    string *retval = new string;
    WeEntityList::iterator  chld;

    *retval = "";
    for (chld = chldList.begin(); chld != chldList.end(); chld++) {
        if (iequals((*chld)->Name(), string("#text"))) {
            *retval += (*chld)->Attr("");
        }
    }

             return(*retval);
}

const string& WeHtmlEntity::OuterText(void)
{
    string *retval = new string;
    WeEntityList::iterator chld;
    WeAttrMap::iterator attr;
    char quote = '"';

    *retval = "<";
    *retval += entityName;
    for (attr = attributes.begin(); attr != attributes.end(); attr++) {
        *retval += " ";
        *retval += attributes.key(attr);
        *retval += "=";
        if (attributes.val(attr) != "") {
            if (attributes.val(attr).find("\\\"") != string::npos) {
                // quoted " found - use the " mark
                quote = '"';
            }
            else {
                quote = '\'';
            }
            *retval += quote;
            *retval += attributes.val(attr);
            *retval += quote;
        }
    }
    *retval += ">";
    if (chldList.size() > 0 || !WeParseNeedToBreakTag(entityName, "")){
        for (chld = chldList.begin(); chld != chldList.end(); chld++) {
            *retval += (*chld)->OuterText();
        }
        *retval += "</";
        *retval += entityName;
        *retval += ">";
    }

             return(*retval);
}

weCmpState WeHtmlEntity::Compare(iweEntity& entity, weCmpMode mode)
{
    if (mode == weCmpDefault) {
        mode = compareMode;
    }
    throw runtime_error("Not implemented");
             return(weCmpLess);
}

WeScannerToken WeHtmlEntity::Parse( string tagName, WeTagScanner& scanner, iweTransport* processor /*= NULL*/ )
{
    WeScannerToken  state;
    WeHtmlEntity    *chld;
    WeInnerText     *txt;
    string           txtAttr;

    if (! tagName.empty()) {
        // for WeDocuments this parameter must be empty
        // and not to overwrite the 'name' property
        entityName = tagName;
    }

    if (processor != NULL) {
        // opts = processor->Options();
    }
    else {
        // if no WeHTTP given - use default options? but no follow the links
        //opts = WeHTTP::weoDefault & (~WeHTTP::weoFollowLinks);
    }
    startPos = scanner.GetPos() - tagName.length();
    if (tagName.length() > 0) {
        startPos--; // if tagName present - skip the "<" symbol
    }
    attributes.clear();
    ClearChildren();
    txt = NULL;
    chld = NULL;
    txtAttr = "";

    while (true)
    {
        state = scanner.GetToken();
parseRestart:
        if (state == wstEof || state == wstError) {
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            break;
        }
        switch(state)
        {
        case wstTagEnd:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            if (iequals(string("form"), string(scanner.GetTagName()))) {
                // right now all form's attributes will be placed to parent entity
//-DBG: printf("FORM CLOSE\n");
                break;
            }
            if (!iequals(entityName, string(scanner.GetTagName()))) {
                // incomplete structure - close this tag
                // and restart parsing on previous level
//-DBG: printf("TAG END: %s (force)\n", entityName.c_str());
                if (!IsParentTag(string(scanner.GetTagName()))) {
                    // if tag is not in paren tree - just skip it
                    break;
                }
            }
            else {
//-DBG: printf("TAG END: %s\n", scanner.GetTagName());
            }
            endPos = scanner.GetPos();
            return state;
        case wstTagStart:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            if (iequals(string("form"), string(scanner.GetTagName()))) {
//-DBG: printf("NEW FORM BEGIN\n");
            }
            if (WeParseNeedToBreakTag(entityName, scanner.GetTagName())) {
                // incomplete structure - close this tag
                // and restart parsing on previous level
//-DBG: printf("TAG END: %s (force)\n", entityName.c_str());
                return state;
            }
            if (iequals(string("form"), string(scanner.GetTagName()))) {
                // right now all form's attributes will be placed to parent entity
                break;
            }
//-DBG: printf("TAG START: %s\n", scanner.GetTagName());
            chld = weHtmlFactory.CreateEntity(scanner.GetTagName(), this);
            if (chld != NULL) {
                WeScannerToken chldState;
                chldList.push_back(chld);
                chldState = chld->Parse(scanner.GetTagName(), scanner, processor);
                if (iequals(string("form"), string(scanner.GetTagName()))) {
                    // form finished?
                    // right now all form's attributes will be placed to parent entity
                    break;
                }
                if (!iequals(chld->Name(), scanner.GetTagName())) {
                    // previous tag was closed incorrectly
                    // restart parsing
                    state = chldState;
                    goto parseRestart;
                }
                chld = NULL;
            }
             break;
        case wstAttr:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
//-DBG: printf("\tATTR: %s=%s\n", scanner.GetAttrName(), scanner.GetValue());
            attributes[scanner.GetAttrName()] = scanner.GetValue();
             break;
        case wstWord:
        case wstSpace:
            if (txt == NULL) {
                txt = new WeInnerText(this);
                txtAttr = "";
            }
            if (state == wstSpace && processor != NULL && processor->IsSet(weoCollapseSpaces)) { //WeHTTP::weoCollapseSpaces)) {
                txtAttr += " ";
            }
            else {
                txtAttr += scanner.GetValue();
            }
            break;
        case wstCommentStart:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            chld = new WeHtmlComment(this);
            break;
        case wstCDataStart:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            chld = new WeCData(this);
            break;
        case wstPiStart:
            if (txt != NULL) {
                txt->Attr("", txtAttr);
                chldList.push_back(txt);
//-DBG: printf("\tTEXT: {%s}\n", txtAttr.c_str());
                txtAttr.clear();
                txt = NULL;
            }
            chld = new WePhpInclude(this);
            break;
        case wstData:
            // force points the '#text' property to avoid other children types break
//-DBG: printf("%s - DATA: %s\n", chld->entityName.c_str(), scanner.GetValue());
            chld->Attr("#text", scanner.GetValue());
            break;
        case wstCommentEnd:
        case wstCDataEnd:
        case wstPiEnd:
            if (chld != NULL) {
                chldList.push_back(chld);
                chld = NULL;
            }
            break;
        default:
            break;
        }
    }

    endPos = scanner.GetPos();
    return state;
}

WeCmpResults* WeHtmlEntity::Diff( iweEntity& cmp, weCmpMode mode )
{
    return NULL;
}

WeInnerText::WeInnerText(iweEntity* prnt /*= NULL*/) :
    WeHtmlEntity(prnt)
{
    entityName = "#text";
    attributes["#text"] = "";
}

WeInnerText::WeInnerText(WeInnerText& entity) :
    WeHtmlEntity()
{
    entityName = "#text";
    attributes["#text"] = entity.attributes["#text"];
}

WeInnerText::~WeInnerText()
{
    // nothing special
}

const string WeInnerText::Attr(string name)
{
    WeAttrMap::iterator it;

    it = attributes.find("#text");
    if (it != attributes.end())
    {
        return attributes.val(it);
    }
    return(*(new string("")));
}

void WeInnerText::Attr(string name, string value)
{
             attributes["#text"] = value;
}

const string& WeInnerText::InnerText(void)
{
    WeAttrMap::iterator it;

    it = attributes.find(string("#text"));
    if (it != attributes.end())
    {
        return attributes.val(it);
    }
             return(*(new string("")));
}

const string& WeInnerText::OuterText(void)
{
    WeAttrMap::iterator it;

    it = attributes.find(string("#text"));
    if (it != attributes.end())
    {
        return attributes.val(it);
    }
    return(*(new string("")));
}

WeCmpResults* WeInnerText::Diff(iweEntity& cmp, weCmpMode mode)
{
    WeInnerText* ptr;
    WeCmpResults* retval = NULL;
    string s1, s2;
    regex repl("\\s+");

    if (mode == weCmpDefault) {
        mode = compareMode;
    }
    try {
        ptr = reinterpret_cast<WeInnerText*>(&cmp);
        s1 = attributes["#text"];
        s2 = ptr->attributes["#text"];
        retval = WeTextDiff(s1, s2, mode);
    } catch (...) {
        if (retval != NULL) {
            delete retval;
        }
        retval = NULL;
    }

    return retval;
}

weCmpState WeInnerText::Compare(iweEntity& cmp, weCmpMode mode)
{
    WeInnerText* ptr;
    string s1, s2;
    weCmpState retval = weCmpNonComparable;
    regex repl("\\s+");

    if (mode == weCmpDefault) {
        mode = compareMode;
    }
    try {
        ptr = reinterpret_cast<WeInnerText*>(&cmp);
        s1 = attributes["#text"];
        s2 = ptr->attributes["#text"];
        if (mode & weCmpCollapseSpace) {
            s1 = regex_replace(s1, repl, " ");
            s2 = regex_replace(s2, repl, " ");
        }
        if (mode & weCmpCaseInsens) {
            is_iless cp_less;
            if (iequals(s1, s2)) {
                retval = weCmpEqual;
            }
            else if(cp_less(s1.c_str(), s2.c_str())) {
                retval = weCmpLess;
            } else {
                retval = weCmpGreather;
            }
        }
        else {
            is_less cp_less;
            if (equals(s1, s2)) {
                retval = weCmpEqual;
            }
            else if(cp_less(s1, s2)) {
                retval = weCmpLess;
            } else {
                retval = weCmpGreather;
            }
        }
    } catch (...) {
        retval = weCmpNonComparable;
    }

    return retval;
}

WeHtmlDocument::WeHtmlDocument(iweEntity* prnt /*= NULL*/) //:
//    WeHtmlEntity(prnt), WeHttpResponse()
{
    entityName = "#document";
//     baseUrl = "";
//     realUrl = "";
//     relocCount = 0;
}

WeHtmlDocument::WeHtmlDocument(WeHtmlDocument& entity) //:
//    WeHtmlEntity(), WeHttpResponse()
{
    entityName = "#document";
    response = NULL;
}

WeHtmlDocument::~WeHtmlDocument(void)
{
    // nothing special for this class, just base classes
}

const string& WeHtmlDocument::InnerText(void)
{
    return OuterText();
}

const string& WeHtmlDocument::OuterText(void)
{
    string *retval = new string;
    WeEntityList::iterator  chld;

    *retval = "";
    for (chld = chldList.begin(); chld != chldList.end(); chld++) {
        *retval += (*chld)->OuterText();
    }

    return(*retval);
}

bool WeHtmlDocument::ParseData(iweResponse* resp, iweTransport* processor /*= NULL*/)
{
    bool retval = false;

    try
    {
        response = dynamic_cast<WeHttpResponse*>(resp);
        WeInStream* stream = response->Data().stream();
        if (stream) {
            WeTagScanner scanner(*stream);
            retval = (Parse("", scanner, processor) != wstError);
        }
        return retval;

    }
    catch (...) { }; // just skip
    return retval;
}

WeBlob& WeHtmlDocument::Data( void )
{
    if (response != NULL) {
        return response->Data();
    }
    throw WeError("WeRefrenceObject::Data - no data linked!");
}

WeCmpResults* WeHtmlDocument::Diff( iweEntity& cmp, weCmpMode mode )
{
    return NULL;
}

weCmpState WeHtmlDocument::Compare( iweEntity& cmp, weCmpMode mode )
{
    return weCmpNonComparable;
}
WeRefrenceObject::WeRefrenceObject(iweEntity* prnt /*= NULL*/) :
    WeHtmlDocument(prnt)
{
}

WeRefrenceObject::WeRefrenceObject( WeRefrenceObject& entity )
{
    throw runtime_error("Not implemented");
}

WeRefrenceObject::~WeRefrenceObject()
{
}

WeScript::WeScript( iweEntity* prnt /*= NULL*/ ) :
    WeRefrenceObject(prnt)
{
}

WeScript::WeScript( WeScript& entity )
{
}

WeScript::~WeScript()
{
}

WeScannerToken WeScript::Parse( string tagName, WeTagScanner& scanner, WeHTTP* processor /*= NULL*/ )
{
    WeScannerToken  state;
    string          txtAttr;
    bool            inOurTag;
    bool            inOtherTag;
    bool            inProcess;

    if (! tagName.empty()) {
        // for WeDocuments this parameter must be empty
        // and not to overwrite the 'name' property
        entityName = tagName;
    }

    if (processor != NULL) {
        //opts = processor->Options();
    }
    else {
        // if no WeHTTP given - use default options? but no follow the links
        // opts = WeHTTP::weoDefault & (~WeHTTP::weoFollowLinks);
    }
    attributes.clear();
    ClearChildren();
    txtAttr = "";
    inOurTag = true;
    inOtherTag = false;
    inProcess = true;

    while (inProcess)
    {
        state = scanner.GetToken();
        if (state == wstEof || state == wstError) {
            inProcess = false;
//            data.assign((char*)txtAttr.c_str());
            break;
        }
        switch(state)
        {
        case wstTagEnd:
            inOurTag = false;
            inOtherTag = false;
            // if it our tag - finish parsing
            if (iequals(scanner.GetTagName(), "script")) {
                // It's the finish
                inProcess = false;
//                data.assign((char*)txtAttr.c_str());
            }
            break;
        case wstTagStart:
            // all TAGs inside script are ignored - it's just a text
            inOurTag = false;
            inOtherTag = true;
            txtAttr += "<";
            txtAttr += scanner.GetTagName();
            break;
        case wstAttr:
            // attributes processed only for our tag, else it's just a text
            if (inOurTag) {
                attributes[scanner.GetAttrName()] = scanner.GetValue();
            }
            else {
                string aval = scanner.GetValue();
                char quote = '"';
                if (aval.find("\\\"") != string::npos) {
                    // \" found - use the " mark
                    quote = '"';
                }
                else {
                    quote = '\'';
                }
                txtAttr += " ";
                txtAttr += scanner.GetAttrName();
                // insert correct attributes quotation
                txtAttr += "=";
                txtAttr += quote;
                txtAttr += aval;
                txtAttr += quote;
            }
            break;

        // all other cases
        case wstWord:
        case wstSpace:
        case wstCommentStart:
        case wstCDataStart:
        case wstPiStart:
        case wstData:
        case wstCommentEnd:
        case wstCDataEnd:
        case wstPiEnd:
        default:
            inOurTag = false;
            if (inOtherTag)
            {
                txtAttr += ">";
            }
            inOtherTag = false;

            if (state == wstSpace && processor->IsSet(weoCollapseSpaces)) { //WeHTTP::weoCollapseSpaces)) {
                txtAttr += " ";
            }
            else {
                txtAttr += scanner.GetValue();
            }
            break;
        }
    }

    return state;
}

bool WeScript::SetEngine( void *engine /*= NULL*/ )
{
    return false;
}

void* WeScript::Execute()
{
    return NULL;
}

---

	©2009 Positive Technologies	Generated on Tue Jul 14 18:27:30 2009 for WebEngine by		1.5.7