see #1548 "Object

de-duplication during import process". Also SourceForge bug
2980566 "Import from hostsfile may cause duplicate Objects" and
3030072 "remove duplicates during any import". Now the program can
optionally re-use existing objects from both Standard Objects and
user-defined libraries when it imports existing firewall
configuration. This works for any firewall platform for which we
support policy import.

doc/ChangeLog

@@ -1,3 +1,14 @@
+2011-03-16  vadim  <vadim@netcitadel.com>
+
+	* Importer.cpp (prepareForDeduplication): fixed #1548 "Object
+	de-duplication during import process". Also SourceForge bug
+	2980566 "Import from hostsfile may cause duplicate Objects" and
+	3030072 "remove duplicates during any import". Now the program can
+	optionally re-use existing objects from both Standard Objects and
+	user-defined libraries when it imports existing firewall
+	configuration. This works for any firewall platform for which we
+	support policy import.
+
 2011-03-14  vadim  <vadim@netcitadel.com>
 
 	* pix.g (named_object_network): see #2223 Implemented import of

src/import/Importer.cpp

@@ -135,6 +135,12 @@ Importer::Importer(FWObject *_lib,
     service_maker = new ServiceObjectMaker(Library::cast(library));
 }
 
+void Importer::prepareForDeduplication()
+{
+    address_maker->prepareForDeduplication(library->getRoot());
+    service_maker->prepareForDeduplication(library->getRoot());
+}
+
 void Importer::run()
 {
     // create and run parsers in derived classes

src/import/Importer.h

@@ -263,7 +263,8 @@ public:
 
     void setFileName(const std::string &fn) { input_file_name = fn; }
     void setPlatform(const std::string &pl) { platform = pl; }
-
+    void prepareForDeduplication();
+    
     // add standard line to rule comment, this adds something like
     // "created during import from <file>, line <line>"
     void addStandardImportComment(libfwbuilder::FWObject *obj,

src/import/objectMaker.cpp

@@ -30,6 +30,7 @@
 #include "fwbuilder/FWObject.h"
 #include "fwbuilder/FWObjectDatabase.h"
 #include "fwbuilder/ICMPService.h"
+#include "fwbuilder/ICMP6Service.h"
 #include "fwbuilder/IPService.h"
 #include "fwbuilder/IPv4.h"
 #include "fwbuilder/IPv6.h"
@@ -40,6 +41,13 @@
 #include "fwbuilder/TCPService.h"
 #include "fwbuilder/TagService.h"
 #include "fwbuilder/UDPService.h"
+#include "fwbuilder/physAddress.h"
+
+#include "fwbuilder/Rule.h"
+#include "fwbuilder/RuleSet.h"
+#include "fwbuilder/FWReference.h"
+#include "fwbuilder/Host.h"
+#include "fwbuilder/FWOptions.h"
 
 #include "QStringListOperators.h"
 
@@ -48,6 +56,9 @@
 
 #include <QStringList>
 
+#include <set>
+
+
 extern int fwbdebug;
 
 using namespace libfwbuilder;
@@ -76,7 +87,7 @@ QString ObjectSignature::toString() const
     if (type_name == CustomService::TYPENAME)
         sig << platform << code << protocol_name;
 
-    if (type_name == ICMPService::TYPENAME)
+    if (type_name == ICMPService::TYPENAME || type_name == ICMP6Service::TYPENAME)
         sig << icmp_type << icmp_code;
 
     if (type_name == IPService::TYPENAME)
@@ -98,6 +109,150 @@ QString ObjectSignature::toString() const
     return sig.join("|-|");
 }
 
+void* ObjectSignature::dispatch(Network *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address = obj->getAddressPtr()->toString().c_str();
+    netmask = obj->getNetmaskPtr()->toString().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(NetworkIPv6 *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address = obj->getAddressPtr()->toString().c_str();
+    netmask = obj->getNetmaskPtr()->toString().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(IPv4 *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address = obj->getAddressPtr()->toString().c_str();
+    netmask = InetAddr::getAllOnes().toString().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(IPv6 *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address = obj->getAddressPtr()->toString().c_str();
+    netmask = InetAddr::getAllOnes(AF_INET6).toString().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(AddressRange *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address_range_start = obj->getRangeStart().toString().c_str();
+    address_range_end = obj->getRangeEnd().toString().c_str();
+    return this;
+}
+
+/*
+ * Note that we do not track "compile time" / "run time" attribute of
+ * the object because on import, only "run time" make sense
+ */
+void* ObjectSignature::dispatch(AddressTable *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address_table_name = obj->getSourceName().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(physAddress *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    address = obj->getPhysAddress().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(IPService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    protocol = obj->getProtocolNumber();
+    fragments = obj->getBool("fragm") || obj->getBool("short_fragm");
+    return this;
+}
+
+void* ObjectSignature::dispatch(ICMPService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    icmp_type = obj->getInt("type");
+    icmp_code = obj->getInt("code");
+    return this;
+}
+
+void* ObjectSignature::dispatch(ICMP6Service *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    icmp_type = obj->getInt("type");
+    icmp_code = obj->getInt("code");
+    return this;
+}
+
+void* ObjectSignature::dispatch(TCPService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    src_port_range_start = obj->getSrcRangeStart();
+    src_port_range_end = obj->getSrcRangeEnd();
+    dst_port_range_start = obj->getDstRangeStart();
+    dst_port_range_end = obj->getDstRangeEnd();
+    established = obj->getEstablished();
+
+    set<TCPService::TCPFlag> flags = obj->getAllTCPFlags();
+    set<TCPService::TCPFlag>::iterator it;
+    for (it=flags.begin(); it!=flags.end(); ++it) flags_comp << *it;
+
+    flags = obj->getAllTCPFlagMasks();
+    for (it=flags.begin(); it!=flags.end(); ++it) flags_mask << *it;
+    return this;
+}
+
+void* ObjectSignature::dispatch(UDPService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    src_port_range_start = obj->getSrcRangeStart();
+    src_port_range_end = obj->getSrcRangeEnd();
+    dst_port_range_start = obj->getDstRangeStart();
+    dst_port_range_end = obj->getDstRangeEnd();
+    return this;
+}
+
+void* ObjectSignature::dispatch(CustomService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    platform = "";
+    code = "";
+    list<string> platforms = obj->getAllKnownPlatforms();
+    foreach(std::string pl, platforms)
+    {
+        platform += pl.c_str();
+        code += obj->getCodeForPlatform(pl).c_str();
+    }
+    protocol_name = obj->getProtocol().c_str();
+    return this;
+}
+
+void* ObjectSignature::dispatch(TagService *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    tag = obj->getStr("tagcode").c_str();
+    return this;
+}
+
+/*
+ * Note that we do not track "compile time" / "run time" attribute of
+ * the object because on import, only "run time" make sense
+ */
+void* ObjectSignature::dispatch(DNSName *obj, void*)
+{
+    type_name = obj->getTypeName().c_str();
+    dns_name = obj->getSourceName().c_str();
+    return this;
+}
+
+
 //****************************************************************
 
 void ObjectMaker::clear()
@@ -143,3 +298,30 @@ FWObject* ObjectMaker::createObject(FWObject *parent,
     o->setName(objName);
     return o;
 }
+
+//****************************************************************
+
+/*
+ * scan the tree starting at @root and use registerObject to build
+ * signatures for all address and service objects in order to be able
+ * to use them on import
+ */
+void ObjectMaker::prepareForDeduplication(FWObject *root)
+{
+    if (RuleSet::cast(root) || Rule::cast(root) ||
+        FWReference::cast(root) ||
+        Host::cast(root) ||
+        FWOptions::cast(root)) return;
+
+    if (Address::cast(root) || Service::cast(root))
+    {
+        ObjectSignature sig;
+        root->dispatch(&sig, (void*)(NULL));
+        registerObject(sig, root);
+    }
+
+    for (FWObject::iterator it=root->begin(); it!=root->end(); ++it)
+    {
+        prepareForDeduplication(*it);
+    }
+}

src/import/objectMaker.h

@@ -26,6 +26,7 @@
 #define _OBJECT_MAKER_H_
 
 #include "fwbuilder/FWObject.h"
+#include "fwbuilder/Dispatch.h"
 
 #include <QString>
 #include <QMap>
@@ -33,7 +34,23 @@
 
 namespace libfwbuilder
 {
+    class AddressRange;
+    class Cluster;
+    class CustomService;
+    class Firewall;
+    class Host;
+    class ICMPService;
+    class IPService;
+    class IPv4;
+    class IPv6;
+    class Interface;
     class Library;
+    class Network;
+    class NetworkIPv6;
+    class TCPService;
+    class TagService;
+    class UDPService;
+    class physAddress;
 };
 
 
@@ -48,7 +65,7 @@ public:
 };
 
 
-class ObjectSignature
+class ObjectSignature : public libfwbuilder::Dispatch
 {
 public:
     QString type_name;
@@ -87,6 +104,25 @@ public:
     QString tag;
 
     QString toString() const;
+
+    // The following methods build signature from given object taking
+    // into account its type
+    virtual void* dispatch(libfwbuilder::Network*, void*);
+    virtual void* dispatch(libfwbuilder::NetworkIPv6*, void*);
+    virtual void* dispatch(libfwbuilder::IPv4*, void*);
+    virtual void* dispatch(libfwbuilder::IPv6*, void*);
+    virtual void* dispatch(libfwbuilder::AddressRange*, void*);
+    virtual void* dispatch(libfwbuilder::AddressTable*, void*);
+    virtual void* dispatch(libfwbuilder::physAddress*, void*);
+    virtual void* dispatch(libfwbuilder::IPService*, void*);
+    virtual void* dispatch(libfwbuilder::ICMPService*, void*);
+    virtual void* dispatch(libfwbuilder::ICMP6Service*, void*);
+    virtual void* dispatch(libfwbuilder::TCPService*, void*);
+    virtual void* dispatch(libfwbuilder::UDPService*, void*);
+    virtual void* dispatch(libfwbuilder::CustomService*, void*);
+    virtual void* dispatch(libfwbuilder::TagService*, void*);
+    virtual void* dispatch(libfwbuilder::DNSName*, void*);
+
 };
 
 class ObjectMaker
@@ -112,13 +148,10 @@ public:
     libfwbuilder::FWObject* createObject(libfwbuilder::FWObject *parent,
                                          const std::string &objType,
                                          const std::string &objName);
-
     
-    void addStandardImportComment(libfwbuilder::FWObject *obj,
-                                  const QString &file_name,
-                                  int line_num);
-
     libfwbuilder::FWObject *getLastCreatedObject() { return last_created; }
+
+    void prepareForDeduplication(libfwbuilder::FWObject *root);
     
 };
 

src/libfwbuilder/src/fwbuilder/CustomService.cpp

@@ -184,6 +184,17 @@ const string CustomService::getCodeForPlatform(const string& platform) const
     return it->second;
 }
 
+list<string> CustomService::getAllKnownPlatforms()
+{
+    list<string> res;
+    map<string, string>::const_iterator i;
+    for (i=codes.begin(); i!=codes.end(); ++i)  
+    {
+        res.push_back( (*i).first);
+    }
+    return res;
+}
+
 void  CustomService::setProtocol(const string& proto)
 {
     protocol = proto;

src/libfwbuilder/src/fwbuilder/CustomService.h

@@ -79,7 +79,8 @@ class CustomService : public Service
     void setCodeForPlatform(const std::string& platform,
                             const std::string& code);
     const std::string getCodeForPlatform(const std::string& platform) const;
-
+    std::list<std::string> getAllKnownPlatforms();
+    
     void setProtocol(const std::string& proto);
     const std::string& getProtocol();
 

src/libgui/importFirewallConfigurationWizard/IC_FirewallNamePage.cpp

@@ -40,7 +40,10 @@ IC_FirewallNamePage::IC_FirewallNamePage(QWidget *parent) : QWizardPage(parent)
     m_dialog = new Ui::IC_FirewallNamePage_q;
     m_dialog->setupUi(this);
 
+    m_dialog->deduplicateOnImport->setChecked(true);
+
     registerField("firewallName*", m_dialog->firewallName);
+    registerField("deduplicate", m_dialog->deduplicateOnImport);
 }
 
 void IC_FirewallNamePage::initializePage()

src/libgui/importFirewallConfigurationWizard/IC_ProgressPage.cpp

@@ -135,6 +135,8 @@ void IC_ProgressPage::initializePage()
         dynamic_cast<ImportFirewallConfigurationWizard*>(wizard())->
         getPlatform();
     QString firewallName = field("firewallName").toString();
+    bool deduplicate = field("deduplicate").toBool();
+
     QStringList *buffer = 
         dynamic_cast<ImportFirewallConfigurationWizard*>(wizard())->
         getBufferPtr();
@@ -142,9 +144,9 @@ void IC_ProgressPage::initializePage()
 
     importer = new ImporterThread(this,
                                   mw->getCurrentLib(),
-                                  *buffer, platform, firewallName, fileName);
+                                  *buffer, platform, firewallName, fileName,
-
+                                  deduplicate);
-
+    
     connect(importer, SIGNAL(destroyed(QObject*)),
             this, SLOT(importerDestroyed(QObject*)));
     connect(importer, SIGNAL(finished()),

src/libgui/importFirewallConfigurationWizard/ImporterThread.cpp

@@ -53,7 +53,8 @@ ImporterThread::ImporterThread(QWidget *ui,
                                const QStringList &buffer,
                                const QString &platform,
                                const QString &firewallName,
-                               const QString &fileName)
+                               const QString &fileName,
+                               bool deduplicate)
 {
     this->lib = lib;
     this->ui = ui;
@@ -61,6 +62,7 @@ ImporterThread::ImporterThread(QWidget *ui,
     this->platform = platform;
     this->firewallName = firewallName;
     this->fileName = fileName;
+    this->deduplicate = deduplicate;
     importer = NULL;
     stopFlag = false;
 }
@@ -94,6 +96,7 @@ void ImporterThread::run()
     {
 
         importer->setFileName(fileName.toUtf8().constData());
+        if (deduplicate) importer->prepareForDeduplication();
 
         try
         {

src/libgui/importFirewallConfigurationWizard/ImporterThread.h

@@ -50,6 +50,7 @@ class ImporterThread : public QThread
     QStringList buffer;
     QString firewallName;
     QString platform;
+    bool deduplicate;
     QWidget *ui;
     libfwbuilder::Firewall *fw;
     bool stopFlag;
@@ -60,7 +61,8 @@ public:
                    const QStringList &buffer,
                    const QString &platform,
                    const QString &firewallName,
-                   const QString &fileName);
+                   const QString &fileName,
+                   bool deduplicate);
     virtual ~ImporterThread();
 
     void run();

src/libgui/importFirewallConfigurationWizard/ic_firewallnamepage_q.ui

@@ -16,8 +16,8 @@
   <property name="title">
    <string>Enter firewall object name</string>
   </property>
-  <layout class="QGridLayout" name="gridLayout">
+  <layout class="QGridLayout" name="gridLayout_2">
-   <item row="0" column="1">
+   <item row="0" column="2">
     <spacer name="verticalSpacer_2">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -33,7 +33,7 @@
      </property>
     </spacer>
    </item>
-   <item row="1" column="0" colspan="2">
+   <item row="1" column="0" colspan="3">
     <layout class="QHBoxLayout" name="horizontalLayout">
      <item>
       <widget class="QLabel" name="label">
@@ -60,7 +60,44 @@
      </item>
     </layout>
    </item>
-   <item row="2" column="0">
+   <item row="2" column="1">
+    <spacer name="verticalSpacer_3">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeType">
+      <enum>QSizePolicy::Fixed</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item row="3" column="0" colspan="3">
+    <layout class="QGridLayout" name="gridLayout">
+     <item row="0" column="0">
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>The program can use objects that already exist in the &quot;Standard Objects&quot; library and user defined libraries to represent addresses and services found in the configuration being imported. This helps avoid duplicate objects. </string>
+       </property>
+       <property name="wordWrap">
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+     <item row="1" column="0">
+      <widget class="QCheckBox" name="deduplicateOnImport">
+       <property name="text">
+        <string>Find and use existing objects</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="4" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>