constcast
/
vermont
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'merge-features'

master
Lothar Braun 2013-09-29 16:43:59 +02:00
parent 61e8041f68 a1b29e84d6
commit acb7a52347
64 changed files with 3926 additions and 4731 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
CMakeLists.txt
View File

@ -96,7 +96,7 @@ OPTION(SUPPORT_MONGO "Enable MongoDB support" OFF)
IF (SUPPORT_MONGO)
FIND_PACKAGE(MONGO REQUIRED)
IF (NOT MONGO_FOUND)
MESSAGE(FATAL_ERROR "Could not find MongoDB libraries.")
MESSAGE(FATAL_ERROR "Could not find MongoDB libraries.")
ENDIF (NOT MONGO_FOUND)
ENDIF (SUPPORT_MONGO)
IF (MONGO_FOUND)
@ -127,6 +127,25 @@ ELSE (MONGO_FOUND)
REMOVE_DEFINITIONS(-DBOOST_FILESYSTEM_VERSION)
ENDIF (MONGO_FOUND)
### Redis
OPTION(SUPPORT_REDIS "Enable Redis support (for flow-inspector integration)" OFF)
IF (SUPPORT_REDIS)
FIND_PACKAGE(Redis REQUIRED)
IF (NOT HIREDIS_FOUND)
MESSAGE(FATAL_ERROR "Could not find hiredis client libraries.")
ENDIF (NOT HIREDIS_FOUND)
ENDIF (SUPPORT_REDIS)
IF (HIREDIS_FOUND)
ADD_DEFINITIONS(-DREDIS_SUPPORT_ENABLED)
INCLUDE_DIRECTORIES(${HIREDIS_INCLUDE_DIR})
TARGET_LINK_LIBRARIES(vermont
${HIREDIS_LIBRARIES}
)
ELSE (HIREDIS_FOUND)
REMOVE_DEFINITIONS(-DREDIS_SUPPORT_ENABLED)
ENDIF (HIREDIS_FOUND)
### boost
FIND_PACKAGE(Boost REQUIRED)
@ -226,12 +245,6 @@ ELSE (DEBUG)
ENDIF (DEBUG)
### IP_HEADER_OFFSET
SET(IP_HEADER_OFFSET 14 CACHE STRING "Start position of the IP header in an ethernet frame in Bytes. This value needs to be adjusted according to the network monitored. The default value is 14 for ethernet devices. Other common values are 4 (BSD loop back device) and 18 (Ethernet VLAN)")
ADD_DEFINITIONS(-DIP_HEADER_OFFSET=${IP_HEADER_OFFSET})
### PCAP_MAX_CAPTURE_LENGTH
SET(PCAP_MAX_CAPTURE_LENGTH 128 CACHE STRING "Maximum PCAP packet capture length (this amount of bytes is always allocated for each packet, the smaller the better!)")

3
cmake/modules/FindOracle.cmake
View File

@ -21,6 +21,7 @@ else(ORACLE_INCLUDE_DIR AND ORACLE_CLIENT_LIBRARIES AND ORACLE_CONNECTION_LIBRAR
/usr/lib/oracle/xe/app/oracle/product/*/client/rdbms/public
/usr/include/oracle/*/client
/opt/oracle/product/*/client/rdbms/public
/opt/oracle/*/client/
)
find_library(ORACLE_CLIENT_LIBRARIES NAMES clntsh libclntsh
@ -28,6 +29,7 @@ else(ORACLE_INCLUDE_DIR AND ORACLE_CLIENT_LIBRARIES AND ORACLE_CONNECTION_LIBRAR
/usr/lib/oracle/xe/app/oracle/product/*/client/lib
/usr/lib/oracle/*/client/lib
/opt/oracle/product/*/client/lib
/opt/oracle/*/client/lib
)
find_library(ORACLE_CONNECTION_LIBRARIES NAMES occi libocci
@ -35,6 +37,7 @@ else(ORACLE_INCLUDE_DIR AND ORACLE_CLIENT_LIBRARIES AND ORACLE_CONNECTION_LIBRAR
/usr/lib/oracle/xe/app/oracle/product/*/client/lib
/usr/lib/oracle/*/client/lib
/opt/oracle/product/*/client/lib
/opt/oracle/*/client/lib
)
if(ORACLE_INCLUDE_DIR AND ORACLE_CLIENT_LIBRARIES AND ORACLE_CONNECTION_LIBRARIES)

42
cmake/modules/FindRedis.cmake Normal file
View File

@ -0,0 +1,42 @@
# - Find Redis client db (hiredis from https://github.com/antirez/hiredis)
# Find the hiredis includes and client library
# This module defines
# HIREDIS_INCLUDE_DIR, where to find hiredis/hiredis.h
# HIREDIS_LIBRARIES, the libraries needed to use redis.
# HIREDIS_FOUND, If false, do not try to use redis.
#
# Copyright (c) 2012, Lothar Braun, <braun@net.in.tum.de>
#
# Add the redis include paths here
IF (HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)
SET (HIREDIS_FOUND TRUE)
ELSE(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)
FIND_PATH(HIREDIS_INCLUDE_DIR hiredis/hiredis.h
/usr/include/
/usr/include/hiredis
/usr/local/include/
/usr/local/include/hiredis
/opt/local/include/
/opt/local/include/hiredis
)
FIND_LIBRARY(HIREDIS_LIBRARIES NAMES hiredis libhiredis
PATHS
/usr/lib
/usr/local/lib
/opt/local/lib
)
IF(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)
SET(HIREDIS_FOUND TRUE)
MESSAGE(STATUS "Found hiredis: ${HIREDIS_INCLUDE_DIR}, ${HIREDIS_LIBRARIES}")
INCLUDE_DIRECTORIES(${HIREDIS_INCLUDE_DIR})
ELSE(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)
SET(HIREDIS_FOUND FALSE)
MESSAGE(STATUS "hiredis client library not found.")
ENDIF(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)
MARK_AS_ADVANCED(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARIES)
ENDIF(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARIES)

8
configs/dbanon.xml
View File

@ -1,5 +1,5 @@
<ipfixConfig>
<ipfixDbReader id="1">
<ipfixDbReaderMySQL id="1">
<host>10.159.5.10</host>
<port>3306</port>
<dbname>test</dbname>
@ -8,7 +8,7 @@
<fullspeed>true</fullspeed>
<timeshift>false</timeshift>
<next>2</next>
</ipfixDbReader>
</ipfixDbReaderMySQL>
<ipfixQueue id="2">
<maxSize>1000</maxSize>
@ -37,7 +37,7 @@
<ipfixPrinter id="7">
</ipfixPrinter>
<ipfixDbWriter id="6">
<ipfixDbWriterMySQL id="6">
<host>10.159.5.10</host>
<port>3306</port>
<dbname>test2</dbname>
@ -58,5 +58,5 @@
<name>lastSwitchedMillis</name>
<name>exporterID</name>
</columns>
</ipfixDbWriter>
</ipfixDbWriterMySQL>
</ipfixConfig>

4
configs/dbwriter.xml
View File

@ -76,7 +76,7 @@
<next>5</next>
</ipfixAggregator>
<ipfixDbWriter id="4">
<ipfixDbWriterMySQL id="4">
<host>127.0.0.1</host>
<port>3306</port>
<dbname>flows_vermont</dbname>
@ -98,7 +98,7 @@
<name>lastSwitchedMillis</name>
<name>exporterID</name>
</columns>
</ipfixDbWriter>
</ipfixDbWriterMySQL>
<ipfixPrinter id="5">
</ipfixPrinter>

4
src/common/ipfixlolib/ipfix_names.c
View File

@ -44,7 +44,9 @@ struct ipfix_identifier ipfixids[] = {
{ IPFIX_TYPEID_egressInterface, IPFIX_LENGTH_egressInterface, 0, "egressInterface" },
{ IPFIX_TYPEID_ipNextHopAsNumber, IPFIX_LENGTH_ipNextHopAsNumber, 0, "ipNextHopAsNumber" },
{ IPFIX_TYPEID_bgpSourceAsNumber, IPFIX_LENGTH_bgpSourceAsNumber, 0, "bgpSourceAsNumber" },
{ IPFIX_TYPEID_bgpSourceAsNumber, IPFIX_LENGTH_bgpSourceAsNumber, IPFIX_PEN_reverse, "revBgpSourceAsNumber" },
{ IPFIX_TYPEID_bgpDestinationAsNumber, IPFIX_LENGTH_bgpDestinationAsNumber, 0, "bgpDestinationAsNumber" },
{ IPFIX_TYPEID_bgpDestinationAsNumber, IPFIX_LENGTH_bgpDestinationAsNumber, IPFIX_PEN_reverse, "revBgpDestinationAsNumber" },
{ IPFIX_TYPEID_bgpNextHopAsNumber, IPFIX_LENGTH_bgpNextHopAsNumber, 0, "bgpNextHopAsNumber" },
{ IPFIX_TYPEID_bgpNextHopIPv4Address, IPFIX_LENGTH_bgpNextHopIPv4Address, 0, "bgpNextHopIPv4Address" },
{ IPFIX_TYPEID_bgpNextHopIPv6Address, IPFIX_LENGTH_bgpNextHopIPv6Address, 0, "bgpNextHopIPv6Address" },
@ -85,10 +87,12 @@ struct ipfix_identifier ipfixids[] = {
{ IPFIX_TYPEID_octetDeltaCount, IPFIX_LENGTH_octetDeltaCount, IPFIX_PEN_reverse, "revOctetDeltaCount" },
{ IPFIX_TYPEID_postOctetDeltaCount, IPFIX_LENGTH_postOctetDeltaCount, 0, "postOctetDeltaCount" },
{ IPFIX_TYPEID_octetTotalCount, IPFIX_LENGTH_octetTotalCount, 0, "octetTotalCount" },
{ IPFIX_TYPEID_octetTotalCount, IPFIX_LENGTH_octetTotalCount, IPFIX_PEN_reverse, "revOctetTotalCount" },
{ IPFIX_TYPEID_packetDeltaCount, IPFIX_LENGTH_packetDeltaCount, 0, "packetDeltaCount" },
{ IPFIX_TYPEID_packetDeltaCount, IPFIX_LENGTH_packetDeltaCount, IPFIX_PEN_reverse, "revPacketDeltaCount" },
{ IPFIX_TYPEID_postPacketDeltaCount, IPFIX_LENGTH_postPacketDeltaCount, 0, "postPacketDeltaCount" },
{ IPFIX_TYPEID_packetTotalCount, IPFIX_LENGTH_packetTotalCount, 0, "packetTotalCount" },
{ IPFIX_TYPEID_packetTotalCount, IPFIX_LENGTH_packetTotalCount, IPFIX_PEN_reverse, "revPacketTotalCount" },
{ IPFIX_TYPEID_droppedOctetDeltaCount, IPFIX_LENGTH_droppedOctetDeltaCount, 0, "droppedOctetDeltaCount" },
{ IPFIX_TYPEID_droppedOctetTotalCount, IPFIX_LENGTH_droppedOctetTotalCount, 0, "droppedOctetTotalCount" },
{ IPFIX_TYPEID_droppedPacketDeltaCount, IPFIX_LENGTH_droppedPacketDeltaCount, 0, "droppedPacketDeltaCount" },

4
src/common/ipfixlolib/ipfixlolib.c
View File

@ -1127,6 +1127,7 @@ static int add_collector_datafile(ipfix_receiving_collector *collector, const ch
collector->ipv4address[0] = '\0';
collector->port_number = 0;
collector->data_socket = -1;
collector->protocol = DATAFILE;
memset(&(collector->addr), 0, sizeof(collector->addr));
collector->last_reconnect_attempt_time = 0;
@ -1145,7 +1146,7 @@ static int add_collector_rawdir(ipfix_receiving_collector *collector, const char
collector->data_socket = -1;
memset(&(collector->addr), 0, sizeof(collector->addr));
collector->last_reconnect_attempt_time = 0;
collector->protocol = RAWDIR;
collector->packet_directory_path = strdup(path);
collector->packets_written = 0;
@ -1375,6 +1376,7 @@ int ipfix_add_collector(ipfix_exporter *exporter, const char *coll_ip4_addr,
);
return -1;
}
#ifdef IPFIXLOLIB_RAWDIR_SUPPORT
/* It is the duty of add_collector_rawdir to set collector->state */
if (proto==RAWDIR) return add_collector_rawdir(collector,coll_ip4_addr);

26
src/modules/CMakeLists.txt
View File

@ -64,13 +64,6 @@ ADD_LIBRARY(modules
ipfix/IpfixCollectorCfg.cpp
ipfix/IpfixCsExporter.cpp
ipfix/IpfixCsExporterCfg.cpp
ipfix/IpfixDbReaderCfg.cpp
ipfix/IpfixDbReaderOracle.cpp
ipfix/IpfixDbReaderOracleCfg.cpp
ipfix/IpfixDbWriterCfg.cpp
ipfix/IpfixDbWriterPgCfg.cpp
ipfix/IpfixDbWriterOracleCfg.cpp
ipfix/IpfixDbWriterMongoCfg.cpp
ipfix/IpfixExporterCfg.cpp
ipfix/IpfixFileWriter.cpp
ipfix/IpfixFileWriterCfg.cpp
@ -97,11 +90,6 @@ ADD_LIBRARY(modules
ipfix/IpfixSender.cpp
ipfix/IpfixRawdirWriter.cpp
ipfix/TemplateBuffer.cpp
ipfix/IpfixDbReader.cpp
ipfix/IpfixDbWriter.cpp
ipfix/IpfixDbWriterPg.cpp
ipfix/IpfixDbWriterOracle.cpp
ipfix/IpfixDbWriterMongo.cpp
ipfix/IpfixRecordDestination.cpp
ipfix/IpfixSampler.cpp
ipfix/IpfixPayloadWriter.cpp
@ -118,6 +106,20 @@ ADD_LIBRARY(modules
ipfix/aggregator/PacketAggregator.cpp
ipfix/aggregator/Rules.cpp
ipfix/aggregator/Rule.cpp
ipfix/database/IpfixDbWriterSQL.cpp
ipfix/database/IpfixDbWriterCfg.cpp
ipfix/database/IpfixDbWriterMongo.cpp
ipfix/database/IpfixDbWriterMongoCfg.cpp
ipfix/database/IpfixDbWriterOracle.cpp
ipfix/database/IpfixDbReaderCfg.cpp
ipfix/database/IpfixDbReader.cpp
ipfix/database/IpfixDbReaderMySQL.cpp
ipfix/database/IpfixDbWriterMySQL.cpp
ipfix/database/IpfixDbWriterPg.cpp
ipfix/database/IpfixDbReaderOracle.cpp
ipfix/database/IpfixFlowInspectorExporterCfg.cpp
ipfix/database/IpfixFlowInspectorExporter.cpp
)

24
src/modules/ConfigManager.cpp
View File

@ -36,21 +36,19 @@
#include "modules/ipfix/IpfixCollectorCfg.h"
#include "modules/ipfix/IpfixExporterCfg.h"
#include "modules/ipfix/IpfixPrinterCfg.h"
#include "modules/ipfix/IpfixDbReaderCfg.h"
#include "modules/ipfix/IpfixDbWriterCfg.h"
#include "modules/ipfix/IpfixFileWriterCfg.hpp"
#include "modules/ipfix/IpfixNetflowExporterCfg.h"
#include "modules/ipfix/IpfixReceiverFileCfg.h"
#include "modules/ipfix/IpfixDbWriterPgCfg.h"
#include "modules/ipfix/IpfixDbWriterOracleCfg.h"
#include "modules/ipfix/IpfixDbReaderOracleCfg.h"
#include "modules/ipfix/IpfixDbWriterMongoCfg.h"
#include "modules/ipfix/IpfixPayloadWriterCfg.h"
#include "modules/ipfix/IpfixSamplerCfg.h"
#include "modules/ipfix/IpfixCsExporterCfg.hpp"
#include "modules/ipfix/NetflowV9ConverterCfg.hpp"
#include "modules/ipfix/aggregator/IpfixAggregatorCfg.h"
#include "modules/ipfix/aggregator/PacketAggregatorCfg.h"
#include "modules/ipfix/database/IpfixDbReaderCfg.h"
#include "modules/ipfix/database/IpfixDbWriterCfg.h"
#include "modules/ipfix/database/IpfixDbWriterMongoCfg.h"
#include "modules/ipfix/database/IpfixFlowInspectorExporterCfg.h"
#include "modules/SensorManagerCfg.h"
#include "modules/analysis/TRWPortscanDetectorCfg.h"
#include "modules/analysis/RBSWormDetectorCfg.h"
@ -100,19 +98,15 @@ Cfg* ConfigManager::configModules[] = {
new P2PDetectorCfg(NULL),
new HostStatisticsCfg(NULL),
new IpfixCsExporterCfg(NULL),
#ifdef DB_SUPPORT_ENABLED
new IpfixDbReaderCfg(NULL),
#if defined(DB_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED)
new IpfixDbWriterCfg(NULL),
#endif
#ifdef PG_SUPPORT_ENABLED
new IpfixDbWriterPgCfg(NULL),
#endif
#ifdef ORACLE_SUPPORT_ENABLED
new IpfixDbWriterOracleCfg(NULL),
new IpfixDbReaderOracleCfg(NULL),
new IpfixDbReaderCfg(NULL),
#endif
#ifdef MONGO_SUPPORT_ENABLED
new IpfixDbWriterMongoCfg(NULL),
#endif
#ifdef REDIS_SUPPORT_ENABLED
new IpfixFlowInspectorExporterCfg(NULL),
#endif
new FlowLenAnalyzerCfg(NULL),
};

12
src/modules/ipfix/FpaPacketGenerator.cpp
View File

@ -116,7 +116,8 @@ void FpaPacketGenerator::onDataRecord(IpfixDataRecord* record)
tv.tv_sec = c.srcTimeEnd/1000;
tv.tv_usec = (c.srcTimeEnd%1000)*1000;
Packet* p = packetManager.getNewInstance();
p->init(tcpDataSegments, tcpSegmentLengths, tv, 0, totlen);
// tcpheader contains a standard 14 bytes ethernet header (see FpaPacketGenerator.h)
p->init(tcpDataSegments, tcpSegmentLengths, tv, 0, totlen, 14);
send(p);
}
totlen = sizeof(tcpHeader)+c.dstPayloadLen;
@ -133,7 +134,8 @@ void FpaPacketGenerator::onDataRecord(IpfixDataRecord* record)
tv.tv_sec = c.dstTimeEnd/1000;
tv.tv_usec = (c.dstTimeEnd%1000)*1000;
Packet* p = packetManager.getNewInstance();
p->init(tcpDataSegments, tcpSegmentLengths, tv, 0, totlen);
// tcpheader contains a standard 14 bytes ethernet header (see FpaPacketGenerator.h)
p->init(tcpDataSegments, tcpSegmentLengths, tv, 0, totlen, 14);
send(p);
}
} else if (c.protocol==17) {
@ -152,7 +154,8 @@ void FpaPacketGenerator::onDataRecord(IpfixDataRecord* record)
tv.tv_sec = c.srcTimeEnd/1000;
tv.tv_usec = (c.srcTimeEnd%1000)*1000;
Packet* p = packetManager.getNewInstance();
p->init(udpDataSegments, udpSegmentLengths, tv, 0, totlen);
// udpheader contains a standard 14 bytes ethernet header (see FpaPacketGenerator.h)
p->init(udpDataSegments, udpSegmentLengths, tv, 0, totlen, 14);
send(p);
}
totlen = sizeof(udpHeader)+c.dstPayloadLen;
@ -170,7 +173,8 @@ void FpaPacketGenerator::onDataRecord(IpfixDataRecord* record)
tv.tv_sec = c.dstTimeEnd/1000;
tv.tv_usec = (c.dstTimeEnd%1000)*1000;
Packet* p = packetManager.getNewInstance();
p->init(udpDataSegments, udpSegmentLengths, tv, 0, totlen);
// udpheader contains a standard 14 bytes ethernet header (see FpaPacketGenerator.h)
p->init(udpDataSegments, udpSegmentLengths, tv, 0, totlen, 14);
send(p);
}

527
src/modules/ipfix/IpfixDbReader.cpp
View File

@ -1,527 +0,0 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef DB_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include "IpfixDbReader.hpp"
#include "IpfixDbCommon.hpp"
#include "common/msg.h"
InstanceManager<IpfixTemplateRecord> IpfixDbReader::templateRecordIM("DbReaderIpfixTemplateRecord", 1);
InstanceManager<IpfixDataRecord> IpfixDbReader::dataRecordIM("DbReaderIpfixDataRecord", 50);
InstanceManager<IpfixTemplateDestructionRecord> IpfixDbReader::templateDestructionRecordIM("DbReaderIpfixTemplateDestructionRecord", 1);
/***** Internal Functions ****************************************************/
void copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type);
/**
* First send a a new template, then send the dataTemplates for all stored
* tables.
*/
void* IpfixDbReader::readFromDB(void* ipfixDbReader_)
{
IpfixDbReader* ipfixDbReader = (IpfixDbReader*)ipfixDbReader_;
ipfixDbReader->registerCurrentThread();
msg(MSG_DIALOG, "IpfixDbReader: Start sending tables");
for(vector<string>::iterator i = ipfixDbReader->tables.begin(); i != ipfixDbReader->tables.end() && !ipfixDbReader->exitFlag; i++) {
boost::shared_ptr<TemplateInfo> templateInfo(new TemplateInfo);
templateInfo->setId = TemplateInfo::IpfixTemplate;
if(ipfixDbReader->dbReaderSendNewTemplate(templateInfo, *i) != 0)
{
msg(MSG_ERROR, "IpfixDbReader: Template error, skip table");
continue;
}
ipfixDbReader->dbReaderSendTable(templateInfo, *i);
ipfixDbReader->dbReaderDestroyTemplate(templateInfo);
}
ipfixDbReader->unregisterCurrentThread();
msg(MSG_DIALOG,"IpfixDbReader: Sending from database is done");
return 0;
}
/**
* Constructs a template from the table data and sends it to all connected
* modules.
*/
int IpfixDbReader::dbReaderSendNewTemplate(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
// reset record length
recordLength = 0;
/**get columnsname of the table*/
if(getColumns(tableName) != 0) {
msg(MSG_ERROR,"IpfixDbReader: Could not get columns for template");
return 1;
}
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
templateInfo->fieldCount++;
templateInfo->fieldInfo = (TemplateInfo::FieldInfo*)realloc(templateInfo->fieldInfo,
sizeof(TemplateInfo::FieldInfo)*templateInfo->fieldCount);
TemplateInfo::FieldInfo* fi = &templateInfo->fieldInfo[templateInfo->fieldCount - 1];
fi->type.id = i->ipfixId;
fi->type.length = i->length;
fi->type.enterprise = 0;
fi->offset = recordLength;
recordLength = recordLength + fi->type.length;
}
/* Pass Data Template to flowSinks */
IpfixTemplateRecord* ipfixRecord = templateRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReader: sent template for table %s", tableName.c_str());
return 0;
}
void copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type) {
switch (type.length) {
case 1:
*(uint8_t*)dest = *(uint8_t*)src;
return;
case 2:
*(uint16_t*)dest = htons(*(uint16_t*)src);
return;
case 4:
*(uint32_t*)dest = htonl(*(uint32_t*)src);
return;
case 8:
*(uint64_t*)dest = htonll(*(uint64_t*)src);
return;
default:
msg(MSG_ERROR, "IpfixDbReader: Uint with length %d unparseable", type.length);
return;
}
}
/**
* Select a given table and get the values by reading
* the database. The Typs of the values from database are
* strings, therefore they must change into IPFIX format
*/
int IpfixDbReader::dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
unsigned offset = 0;
uint64_t delta = 0; // 64 bit to avoid castings in the case of flowStartMilliseconds
uint32_t flowTime = 0; // in seconds, so 32 bit are sufficient
uint32_t lastFlowTime = 0;
uint64_t tmp;
bool first = true;
unsigned j = 0;
string query = "SELECT " + columnNames + " FROM " + tableName;
// at full speed, we do not make time shifts or reorder
if(fullspeed)
timeshift = false; // timeshift disabled in fullspeed mode
else
query = query + orderBy;
msg(MSG_VDEBUG, "IpfixDbReader: SQL query: %s", query.c_str());
if(mysql_query(conn, query.c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbReader: Select on table failed. Error: %s",
mysql_error(conn));
return 1;
}
dbResult = mysql_store_result(conn);
msg(MSG_INFO,"IpfixDbReader: Start sending records from table %s", tableName.c_str());
while((dbRow = mysql_fetch_row(dbResult)) && !exitFlag) {
if (first) {
j = 0;
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); ++i) {
if(i->ipfixId == IPFIX_TYPEID_flowEndSeconds) {
delta = time(NULL) - atoll(dbRow[j]);
lastFlowTime = atoll(dbRow[j]) + delta;
first = false;
break;
} else if(i->ipfixId == IPFIX_TYPEID_flowEndMilliSeconds) {
delta = time(NULL) - atoll(dbRow[j])/1000;
lastFlowTime = atoll(dbRow[j])/1000 + delta;
first = false;
break;
}
j++;
}
if (first) {
msg(MSG_ERROR, "IpfixDbReader: no flowEndSeconds column in table");
mysql_free_result(dbResult);
return 1;
}
if (timeshift)
msg(MSG_DEBUG, "IpfixDbReader: time shift is %d seconds", delta);
}
// build new record
boost::shared_array<IpfixRecord::Data> data(new IpfixRecord::Data[recordLength]);
offset = 0;
j = 0;
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); ++i) {
switch(i->ipfixId) {
case IPFIX_TYPEID_flowEndSeconds:
flowTime = atoll(dbRow[j]) + delta;
case IPFIX_TYPEID_flowStartSeconds:
tmp = atoll(dbRow[j]);
// do time shift if required
if(timeshift)
tmp += delta;
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
flowTime = atoll(dbRow[j])/1000 + delta;
case IPFIX_TYPEID_flowStartMilliSeconds:
tmp = atoll(dbRow[j]);
// do time shift if required
if(timeshift)
tmp += 1000*delta;
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_destinationIPv4Address:
case IPFIX_TYPEID_sourceIPv4Address:
case IPFIX_TYPEID_sourceTransportPort:
case IPFIX_TYPEID_destinationTransportPort:
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_classOfServiceIPv4:
tmp = atoll(dbRow[j]);
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
}
j++;
}
/** according to flowstarttime wait for sending the record*/
if(!fullspeed && (flowTime != lastFlowTime)) {
time_t t = time(NULL);
if (t > (int)flowTime) {
msg(MSG_ERROR, "IpfixDbReader: Sending flows too slowly");
} else {
sleep (flowTime - t);
}
lastFlowTime = flowTime;
}
IpfixDataRecord* ipfixRecord = dataRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
ipfixRecord->dataLength = offset; // = recordLength
ipfixRecord->message = data;
ipfixRecord->data = data.get();
send(ipfixRecord);
msg(MSG_VDEBUG,"IpfixDbReader: Record sent");
}
mysql_free_result(dbResult);
if(!exitFlag)
msg(MSG_INFO,"IpfixDbReader: Sending from table %s done", tableName.c_str());
else
msg(MSG_INFO,"IpfixDbReader: Sending from table %s aborted", tableName.c_str());
return 0;
}
int IpfixDbReader::dbReaderDestroyTemplate(boost::shared_ptr<TemplateInfo> templateInfo)
{
IpfixTemplateDestructionRecord* ipfixRecord = templateDestructionRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReader: Template destroyed");
return 0;
}
/**
* get all tables in database that matches with the wildcard "h\_%"
**/
int IpfixDbReader::getTables()
{
const char* wild = "h\\_%";
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
dbResult = mysql_list_tables(conn, wild);
if(dbResult == 0) {
msg(MSG_FATAL,"IpfixDbReader: There are no flow tables in database");
} else {
while((dbRow = mysql_fetch_row(dbResult))) {
tables.push_back(string(dbRow[0]));
msg(MSG_VDEBUG, "IpfixDbReader: table %s", tables.back().c_str());
}
}
mysql_free_result(dbResult);
return 0;
}
/**
* Get the names of columns
*/
int IpfixDbReader::getColumns(const string& tableName)
{
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
string query = "SHOW COLUMNS FROM " + tableName;
msg(MSG_VDEBUG, "IpfixDbReader: SQL query: %s", query.c_str());
if(mysql_query(conn, query.c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbReader: Show columns on table %s failed. Error: %s",
tableName.c_str(), mysql_error(conn));
return 1;
}
dbResult = mysql_store_result(conn);
if(dbResult == 0) {
msg(MSG_FATAL,"IpfixDbReader: There are no Columns in the table");
return 1;
}
// TODO: don't we have to free the result of mysql_fetch_row?????
columns.clear();
columnNames = "";
orderBy = "";
bool haveFirstMillis = false;
bool haveLastMillis = false;
while((dbRow = mysql_fetch_row(dbResult))) {
bool found = true;
if(strcmp(dbRow[0], CN_dstIP) == 0) {
columnNames = columnNames + ", " + CN_dstIP;
columnDB tmp = {IPFIX_TYPEID_destinationIPv4Address, IPFIX_LENGTH_destinationIPv4Address};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_srcIP) == 0) {
columnNames = columnNames + ", " + CN_srcIP;
columnDB tmp = {IPFIX_TYPEID_sourceIPv4Address, IPFIX_LENGTH_sourceIPv4Address};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_dstPort) == 0) {
columnNames = columnNames + ", " + CN_dstPort;
columnDB tmp = {IPFIX_TYPEID_destinationTransportPort, IPFIX_LENGTH_destinationTransportPort};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_srcPort) == 0) {
columnNames = columnNames + ", " + CN_srcPort;
columnDB tmp = {IPFIX_TYPEID_sourceTransportPort, IPFIX_LENGTH_sourceTransportPort};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_proto) == 0) {
columnNames = columnNames + ", " + CN_proto;
columnDB tmp = {IPFIX_TYPEID_protocolIdentifier, IPFIX_LENGTH_protocolIdentifier};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_dstTos) == 0) {
columnNames = columnNames + ", " + CN_dstTos;
columnDB tmp = {IPFIX_TYPEID_classOfServiceIPv4, IPFIX_LENGTH_classOfServiceIPv4};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_bytes) == 0) {
columnNames = columnNames + ", " + CN_bytes;
columnDB tmp = {IPFIX_TYPEID_octetDeltaCount, IPFIX_LENGTH_octetDeltaCount};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_pkts) == 0) {
columnNames = columnNames + ", " + CN_pkts;
columnDB tmp = {IPFIX_TYPEID_packetDeltaCount, IPFIX_LENGTH_packetDeltaCount};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_firstSwitched) == 0) {
columnNames = columnNames + ", " + CN_firstSwitched;
columnDB tmp = {IPFIX_TYPEID_flowStartSeconds, IPFIX_LENGTH_flowStartSeconds};
columns.push_back(tmp);
} else if(strcmp(dbRow[0], CN_lastSwitched) == 0) {
columnNames = columnNames + ", " + CN_lastSwitched;
columnDB tmp = {IPFIX_TYPEID_flowEndSeconds, IPFIX_LENGTH_flowEndSeconds};
columns.push_back(tmp);
orderBy = " ORDER BY ";
orderBy.append(CN_lastSwitched);
} else if(strcmp(dbRow[0], CN_firstSwitchedMillis) == 0) {
haveFirstMillis = true;
} else if(strcmp(dbRow[0], CN_lastSwitchedMillis) == 0) {
haveLastMillis = true;
} else if(strcmp(dbRow[0], CN_exporterID) != 0) {
msg(MSG_INFO, "IpfixDbReader: Unsupported column: %s", dbRow[0]);
found = false;
}
if(found)
msg(MSG_VDEBUG, "IpfixDbReader: column %s (%d)", dbRow[0], columns.back().ipfixId);
}
// if we have found seconds and milliseconds, forge the columns to get flowStart/EndMilliseconds
string::size_type pos;
if(haveFirstMillis && (pos = columnNames.find(CN_firstSwitched, 0)) != string::npos) {
columnNames.insert(pos, "+1000*");
columnNames.insert(pos, CN_firstSwitchedMillis);
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
if(i->ipfixId == IPFIX_TYPEID_flowStartSeconds) {
i->ipfixId = IPFIX_TYPEID_flowStartMilliSeconds;
i->length = IPFIX_LENGTH_flowStartMilliSeconds;
break;
}
}
}
if(haveLastMillis && (pos = columnNames.find(CN_lastSwitched, 0)) != string::npos) {
columnNames.insert(pos, "+1000*");
columnNames.insert(pos, CN_lastSwitchedMillis);
orderBy = " ORDER BY ";
orderBy.append(CN_lastSwitchedMillis);
orderBy.append("+1000*");
orderBy.append(CN_lastSwitched);
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
if(i->ipfixId == IPFIX_TYPEID_flowEndSeconds) {
i->ipfixId = IPFIX_TYPEID_flowEndMilliSeconds;
i->length = IPFIX_LENGTH_flowEndMilliSeconds;
break;
}
}
}
if(columnNames != "")
columnNames.erase(0,2);
mysql_free_result(dbResult);
return 0;
}
int IpfixDbReader::connectToDb(
const string& hostName, const string& dbName,
const string& userName, const string& password,
unsigned int port)
{
/** get the mysl init handle*/
conn = mysql_init(0);
if(conn == 0) {
msg(MSG_FATAL,"IpfixDbReader: Get MySQL connect handle failed. Error: %s",
mysql_error(conn));
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReader: mysql init successful");
}
/**Connect to Database*/
if (!mysql_real_connect(conn, hostName.c_str(), userName.c_str(),password.c_str(),
0, port, 0, 0)) {
msg(MSG_FATAL,"IpfixDbReader: Connection to database failed. Error: %s",
mysql_error(conn));
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReader: successfully connected to database");
}
/** use database with dbName **/
if(mysql_select_db(conn, dbName.c_str()) !=0) {
msg(MSG_FATAL,"IpfixDbReader: Database %s not selectable", dbName.c_str());
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReader: Database %s selected", dbName.c_str());
}
return 0;
}
/***** Exported Functions ****************************************************/
/**
* Starts or resumes database
* @param ipfixDbReader handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReader::performStart()
{
thread.run(this);
}
/**
* Temporarily pauses database
* @param ipfixDbReader handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReader::performShutdown()
{
thread.join();
}
/**
* Frees memory used by an ipfixDbReader
* @param ipfixDbWriter handle obtained by calling @c createipfixDbReader()
*/
IpfixDbReader::~IpfixDbReader() {
mysql_close(conn);
}
/**
* Creates a new ipfixDbReader. Do not forget to call @c startipfixDbReader() to begin reading from Database
* @return handle to use when calling @c destroyipfixDbRreader()
*/
IpfixDbReader::IpfixDbReader(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint16_t observationDomainId,
bool timeshift, bool fullspeed)
: timeshift(timeshift), fullspeed(fullspeed), thread(readFromDB)
{
srcId.reset(new IpfixRecord::SourceID);
srcId->observationDomainId = observationDomainId;
srcId->exporterAddress.len = 0;
srcId->exporterPort = 0;
srcId->receiverPort = 0;
srcId->protocol = 0;
srcId->fileDescriptor = 0;
if (connectToDb(hostname, dbname, username, password, port)) {
THROWEXCEPTION("IpfixDbReader creation failed");
}
/** get tables of the database*/
if(getTables() != 0) {
msg(MSG_ERROR,"IpfixDbReader: Error in function getTables");
THROWEXCEPTION("IpfixDbReader creation failed");
}
if(fullspeed && timeshift)
msg(MSG_DIALOG, "IpfixDbReader: timeshift configured, but disabled in fullspeed mode");
}
#endif

721
src/modules/ipfix/IpfixDbReaderOracle.cpp
View File

@ -1,721 +0,0 @@
/*
* IPFIX Database Reader for Oracle DBs
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006-2012 Lothar Braun
* Copyright (C) 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include <sstream>
#include "IpfixDbReaderOracle.hpp"
#include "IpfixDbCommon.hpp"
#include "common/msg.h"
InstanceManager<IpfixTemplateRecord> IpfixDbReaderOracle::templateRecordIM("DbReaderIpfixTemplateRecord", 1);
InstanceManager<IpfixDataRecord> IpfixDbReaderOracle::dataRecordIM("DbReaderIpfixDataRecord", 50);
InstanceManager<IpfixTemplateDestructionRecord> IpfixDbReaderOracle::templateDestructionRecordIM("DbReaderIpfixTemplateDestructionRecord", 1);
/***** Internal Functions ****************************************************/
void copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type);
/**
* First send a a new template, then send the dataTemplates for all stored
* tables.
*/
void* IpfixDbReaderOracle::readFromDB(void* ipfixDbReader_)
{
IpfixDbReaderOracle* ipfixDbReader = (IpfixDbReaderOracle*)ipfixDbReader_;
ipfixDbReader->registerCurrentThread();
msg(MSG_DIALOG, "IpfixDbReaderOracle: Start sending tables");
for(vector<string>::iterator i = ipfixDbReader->tables.begin(); i != ipfixDbReader->tables.end() && !ipfixDbReader->exitFlag; i++) {
// check if table can contain flows between firstFlowTime
// and lastFlowTime (in case these are set).
// Only replay traffic from tables that potentially contain flows
// from this time interval. If startTime == 0, all flows from the
// beginning of the database will be read. If endTime == 0, all flows
// until the end of the last table will be read.
if (ipfixDbReader->firstFlowTime != 0 || ipfixDbReader->lastFlowTime) {
if (!ipfixDbReader->isTableBetweenTimestamps(*i, ipfixDbReader->firstFlowTime, ipfixDbReader->lastFlowTime))
continue;
}
boost::shared_ptr<TemplateInfo> templateInfo(new TemplateInfo);
templateInfo->setId = TemplateInfo::IpfixTemplate;
if(ipfixDbReader->dbReaderSendNewTemplate(templateInfo, *i) != 0)
{
msg(MSG_ERROR, "IpfixDbReaderOracle: Template error, skip table");
continue;
}
ipfixDbReader->dbReaderSendTable(templateInfo, *i);
ipfixDbReader->dbReaderDestroyTemplate(templateInfo);
}
ipfixDbReader->unregisterCurrentThread();
msg(MSG_DIALOG,"IpfixDbReaderOracle: Sending from database is done");
return 0;
}
/**
* Constructs a template from the table data and sends it to all connected
* modules.
*/
int IpfixDbReaderOracle::dbReaderSendNewTemplate(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
// reset record length
recordLength = 0;
/**get columnsname of the table*/
if(getColumns(tableName) != 0) {
msg(MSG_ERROR,"IpfixDbReaderOracle: Could not get columns for template");
return 1;
}
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
templateInfo->fieldCount++;
templateInfo->fieldInfo = (TemplateInfo::FieldInfo*)realloc(templateInfo->fieldInfo,
sizeof(TemplateInfo::FieldInfo)*templateInfo->fieldCount);
TemplateInfo::FieldInfo* fi = &templateInfo->fieldInfo[templateInfo->fieldCount - 1];
fi->type.id = i->ipfixId;
fi->type.length = i->length;
fi->type.enterprise = 0;
fi->offset = recordLength;
recordLength = recordLength + fi->type.length;
}
/* Pass Data Template to flowSinks */
IpfixTemplateRecord* ipfixRecord = templateRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReaderOracle: sent template for table %s", tableName.c_str());
return 0;
}
void copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type) {
switch (type.length) {
case 1:
*(uint8_t*)dest = *(uint8_t*)src;
return;
case 2:
*(uint16_t*)dest = htons(*(uint16_t*)src);
return;
case 4:
*(uint32_t*)dest = htonl(*(uint32_t*)src);
return;
case 8:
*(uint64_t*)dest = htonll(*(uint64_t*)src);
return;
default:
msg(MSG_ERROR, "IpfixDbReaderOracle: Uint with length %d unparseable", type.length);
return;
}
}
/**
* Select a given table and get the values by reading
* the database. The Typs of the values from database are
* strings, therefore they must change into IPFIX format
*/
int IpfixDbReaderOracle::dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
std::ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
unsigned offset = 0;
uint64_t delta = 0; // 64 bit to avoid castings in the case of flowStartMilliseconds
uint32_t flowTime = 0; // in seconds, so 32 bit are sufficient
uint32_t lastFlowTime = 0;
uint64_t tmp;
bool first = true;
unsigned j = 0;
msg(MSG_VDEBUG, "IpfixDbReaderOracle: Sending table %s", tableName.c_str());
sql << "SELECT " << columnNames << " FROM "<< tableName;
if (firstFlowTime || lastFlowTime) {
sql << whereClause;
}
// at full speed, we do not make time shifts or reorder
if(fullspeed)
timeshift = false; // timeshift disabled in fullspeed mode
else
sql << orderBy;
// create the oracle statement
try {
stmt = con->createStatement(sql.str());
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
//msg(MSG_VDEBUG, "IpfixDbReaderOracle: SQL query: %s", query.c_str());
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Table %s was empty!", tableName.c_str());
return 1;
}
msg(MSG_INFO,"IpfixDbReaderOracle: Start sending records from table %s", tableName.c_str());
try {
while((rs->next()) && !exitFlag) {
if (first) {
j = 1;
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); ++i) {
if(i->ipfixId == IPFIX_TYPEID_flowEndSeconds) {
delta = time(NULL) - atoll(rs->getString(j).c_str());
lastFlowTime = atoll(rs->getString(j).c_str()) + delta;
first = false;
break;
} else if(i->ipfixId == IPFIX_TYPEID_flowEndMilliSeconds) {
delta = time(NULL) - atoll(rs->getString(j).c_str())/1000;
lastFlowTime = atoll(rs->getString(j).c_str())/1000 + delta;
first = false;
break;
}
j++;
}
if (first) {
msg(MSG_ERROR, "IpfixDbReaderOracle: no flowEndSeconds or floweEndMilliSeconds column in table %s", tableName.c_str());
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
return 1;
}
if (timeshift)
msg(MSG_DEBUG, "IpfixDbReaderOracle: time shift is %d seconds", delta);
}
// build new record
boost::shared_array<IpfixRecord::Data> data(new IpfixRecord::Data[recordLength]);
offset = 0;
j = 0;
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); ++i) {
switch(i->ipfixId) {
case IPFIX_TYPEID_flowEndSeconds:
flowTime = atoll(rs->getString(j + 1).c_str()) + delta;
case IPFIX_TYPEID_flowStartSeconds:
tmp = atoll(rs->getString(j + 1).c_str());
// do time shift if required
if(timeshift)
tmp += delta;
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
flowTime = atoll(rs->getString(j + 1).c_str())/1000 + delta;
case IPFIX_TYPEID_flowStartMilliSeconds:
tmp = atoll(rs->getString(j + 1).c_str());
// do time shift if required
if(timeshift)
tmp += 1000*delta;
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_destinationIPv4Address:
case IPFIX_TYPEID_sourceIPv4Address:
case IPFIX_TYPEID_sourceTransportPort:
case IPFIX_TYPEID_destinationTransportPort:
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_classOfServiceIPv4:
tmp = atoll(rs->getString(j + 1).c_str());
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
break;
}
j++;
}
/** according to flowstarttime wait for sending the record*/
if(!fullspeed && (flowTime != lastFlowTime)) {
time_t t = time(NULL);
if (t > (int)flowTime) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Sending flows too slowly");
} else {
sleep (flowTime - t);
}
lastFlowTime = flowTime;
}
IpfixDataRecord* ipfixRecord = dataRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
ipfixRecord->dataLength = offset; // = recordLength
ipfixRecord->message = data;
ipfixRecord->data = data.get();
send(ipfixRecord);
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught generic SQL exception");
return 1;
}
if(!exitFlag)
msg(MSG_INFO,"IpfixDbReaderOracle: Sending from table %s done", tableName.c_str());
else
msg(MSG_INFO,"IpfixDbReaderOracle: Sending from table %s aborted", tableName.c_str());
return 0;
}
int IpfixDbReaderOracle::dbReaderDestroyTemplate(boost::shared_ptr<TemplateInfo> templateInfo)
{
IpfixTemplateDestructionRecord* ipfixRecord = templateDestructionRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReaderOracle: Template destroyed");
return 0;
}
/**
* get all tables in database that matches with the wildcard "h\_%"
**/
int IpfixDbReaderOracle::getTables()
{
std::ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT table_name FROM user_tables WHERE table_name LIKE 'H_%' ORDER BY table_name ASC";
// create the oracle statement
try {
stmt = con->createStatement(sql.str());
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
//msg(MSG_VDEBUG, "IpfixDbReaderOracle: SQL query: %s", query.c_str());
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Found no flow tables!");
return 1;
}
try {
while((rs->next()) && !exitFlag) {
tables.push_back(rs->getString(1));
msg(MSG_VDEBUG, "IpfixDbReaderOracle: table %s", tables.back().c_str());
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught generic SQL exception");
return 1;
}
return 0;
}
/**
* Get the names of columns
*/
int IpfixDbReaderOracle::getColumns(const string& tableName)
{
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
string query = "SELECT column_name FROM cols WHERE table_name = '" + tableName + "'";
// create the oracle statement
try {
stmt = con->createStatement(query);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
//msg(MSG_VDEBUG, "IpfixDbReaderOracle: SQL query: %s", query.c_str());
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Flow tables do not have columns??");
return 1;
}
columns.clear();
columnNames = "";
orderBy = "";
whereClause = "";
bool haveFirstMillis = false;
bool haveLastMillis = false;
try {
while((rs->next()) && !exitFlag) {
bool found = true;
if(strcasecmp(rs->getString(1).c_str(), CN_dstIP) == 0) {
columnNames = columnNames + ", " + CN_dstIP;
columnDB tmp = {IPFIX_TYPEID_destinationIPv4Address, IPFIX_LENGTH_destinationIPv4Address};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_srcIP) == 0) {
columnNames = columnNames + ", " + CN_srcIP;
columnDB tmp = {IPFIX_TYPEID_sourceIPv4Address, IPFIX_LENGTH_sourceIPv4Address};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_dstPort) == 0) {
columnNames = columnNames + ", " + CN_dstPort;
columnDB tmp = {IPFIX_TYPEID_destinationTransportPort, IPFIX_LENGTH_destinationTransportPort};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_srcPort) == 0) {
columnNames = columnNames + ", " + CN_srcPort;
columnDB tmp = {IPFIX_TYPEID_sourceTransportPort, IPFIX_LENGTH_sourceTransportPort};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_proto) == 0) {
columnNames = columnNames + ", " + CN_proto;
columnDB tmp = {IPFIX_TYPEID_protocolIdentifier, IPFIX_LENGTH_protocolIdentifier};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_dstTos) == 0) {
columnNames = columnNames + ", " + CN_dstTos;
columnDB tmp = {IPFIX_TYPEID_classOfServiceIPv4, IPFIX_LENGTH_classOfServiceIPv4};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_bytes) == 0) {
columnNames = columnNames + ", " + CN_bytes;
columnDB tmp = {IPFIX_TYPEID_octetDeltaCount, IPFIX_LENGTH_octetDeltaCount};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_pkts) == 0) {
columnNames = columnNames + ", " + CN_pkts;
columnDB tmp = {IPFIX_TYPEID_packetDeltaCount, IPFIX_LENGTH_packetDeltaCount};
columns.push_back(tmp);
} else if(strcasecmp(rs->getString(1).c_str(), CN_firstSwitched) == 0) {
columnNames = columnNames + ", " + CN_firstSwitched;
columnDB tmp = {IPFIX_TYPEID_flowStartSeconds, IPFIX_LENGTH_flowStartSeconds};
columns.push_back(tmp);
std::stringstream out;
out << " WHERE " << CN_firstSwitched << " >= " << firstFlowTime;
if (lastFlowTime) {
out << " AND " << CN_firstSwitched << " <= " << lastFlowTime;
}
whereClause = out.str();
} else if(strcasecmp(rs->getString(1).c_str(), CN_lastSwitched) == 0) {
columnNames = columnNames + ", " + CN_lastSwitched;
columnDB tmp = {IPFIX_TYPEID_flowEndSeconds, IPFIX_LENGTH_flowEndSeconds};
columns.push_back(tmp);
orderBy = " ORDER BY ";
orderBy.append(CN_lastSwitched);
} else if(strcasecmp(rs->getString(1).c_str(), CN_firstSwitchedMillis) == 0) {
haveFirstMillis = true;
} else if(strcasecmp(rs->getString(1).c_str(), CN_lastSwitchedMillis) == 0) {
haveLastMillis = true;
} else if(strcasecmp(rs->getString(1).c_str(), CN_exporterID) != 0) {
msg(MSG_INFO, "IpfixDbReaderOracle: Unsupported column: %s", rs->getString(1).c_str());
found = false;
}
if(found)
msg(MSG_VDEBUG, "IpfixDbReaderOracle: column %s (%d)", rs->getString(1).c_str(), columns.back().ipfixId);
}
// if we have found seconds and milliseconds, forge the columns to get flowStart/EndMilliseconds
string::size_type pos;
if(haveFirstMillis && (pos = columnNames.find(CN_firstSwitched, 0)) != string::npos) {
columnNames.insert(pos, "+1000*");
columnNames.insert(pos, CN_firstSwitchedMillis);
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
if(i->ipfixId == IPFIX_TYPEID_flowStartSeconds) {
i->ipfixId = IPFIX_TYPEID_flowStartMilliSeconds;
i->length = IPFIX_LENGTH_flowStartMilliSeconds;
break;
}
}
}
if(haveLastMillis && (pos = columnNames.find(CN_lastSwitched, 0)) != string::npos) {
columnNames.insert(pos, "+1000*");
columnNames.insert(pos, CN_lastSwitchedMillis);
orderBy = " ORDER BY ";
orderBy.append(CN_lastSwitchedMillis);
orderBy.append("+1000*");
orderBy.append(CN_lastSwitched);
for(vector<columnDB>::iterator i = columns.begin(); i != columns.end(); i++) {
if(i->ipfixId == IPFIX_TYPEID_flowEndSeconds) {
i->ipfixId = IPFIX_TYPEID_flowEndMilliSeconds;
i->length = IPFIX_LENGTH_flowEndMilliSeconds;
break;
}
}
}
if(columnNames != "")
columnNames.erase(0,2);
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught generic SQL exception");
return 1;
}
return 0;
}
int IpfixDbReaderOracle::connectToDb(
const string& hostName, const string& dbName,
const string& userName, const string& password,
unsigned int port)
{
dbError = true;
// try to close connection in case we still have one
// from a previous try
if (con) {
env->terminateConnection(con);
}
msg(MSG_DEBUG, "IpfixDbReaderOracle: Creating environment.");
try {
env = oracle::occi::Environment::createEnvironment(oracle::occi::Environment::DEFAULT);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error while creating environment: %s.", ex.getMessage().c_str());
msg(MSG_FATAL, "IpfixDbReaderOracle: Did you configure your Oracle environment?");
return -1;
}
msg(MSG_DEBUG, "IpfixDbReaderOracle: Trying to connect to database ...");
try
{
char dbLogon[256];
sprintf(dbLogon, "%s:%u/%s", hostName.c_str(), port, dbName.c_str());
con = env->createConnection(userName, password, dbLogon);
} catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbReaderOracle: Oracle connect failed. Error: %s", ex.getMessage().c_str());
return 1;
}
msg(MSG_DEBUG,"IpfixDbReaderOracle: Successfully connected to Oracle DB");
return 0;
}
/***** Exported Functions ****************************************************/
/**
* Starts or resumes database
* @param ipfixDbReaderOracle handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReaderOracle::performStart()
{
thread.run(this);
}
/**
* Temporarily pauses database
* @param ipfixDbReader handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReaderOracle::performShutdown()
{
thread.join();
}
/**
* Frees memory used by an ipfixDbReader
* @param ipfixDbWriter handle obtained by calling @c createipfixDbReader()
*/
IpfixDbReaderOracle::~IpfixDbReaderOracle() {
env->terminateConnection(con);
oracle::occi::Environment::terminateEnvironment(env);
}
/**
* Creates a new ipfixDbReader. Do not forget to call @c startipfixDbReader() to begin reading from Database
* @return handle to use when calling @c destroyipfixDbRreader()
*/
IpfixDbReaderOracle::IpfixDbReaderOracle(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint16_t observationDomainId,
bool timeshift, bool fullspeed, uint32_t startTime, uint32_t endTime)
: timeshift(timeshift), fullspeed(fullspeed), firstFlowTime(startTime), lastFlowTime(endTime), thread(readFromDB)
{
srcId.reset(new IpfixRecord::SourceID);
srcId->observationDomainId = observationDomainId;
srcId->exporterAddress.len = 0;
srcId->exporterPort = 0;
srcId->receiverPort = 0;
srcId->protocol = 0;
srcId->fileDescriptor = 0;
if (fullspeed) {
msg(MSG_DEBUG, "IpfixDbReaderOracle: found fullspeed in configuration. Pushing flows as fast as possible.");
}
if (connectToDb(hostname, dbname, username, password, port)) {
THROWEXCEPTION("IpfixDbReaderOracle creation failed");
}
/** get tables of the database*/
if(getTables() != 0) {
msg(MSG_ERROR,"IpfixDbReaderOracle: Error in function getTables");
THROWEXCEPTION("IpfixDbReaderOracle creation failed");
}
if(fullspeed && timeshift)
msg(MSG_DIALOG, "IpfixDbReaderOracle: timeshift configured, but disabled in fullspeed mode");
}
bool IpfixDbReaderOracle::isTableBetweenTimestamps(const string& tablename, uint32_t start, uint32_t end)
{
uint32_t tableStartTime, tableEndTime;
std::stringstream str(tablename);
char tmp[5];
uint32_t timeStamp;
uint32_t time;
struct tm tableTime;
bzero(&tableTime, sizeof(struct tm));
// the table string lookes like H_20120202_10_0 and always
// matches a length of 15 bytes;
if (tablename.size() != 15) {
msg(MSG_DEBUG, "IpfixDbReaderOracle: Found a table name \"%s\" which does not match the expected table name size of 15 bytes.", tablename.c_str());
return false;
}
// get H_
str.get(tmp, 3);
if (std::string(tmp) != "H_") {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found table that does not begin with static prefix \"H_\" but with prefix \"%s\": %s", tmp, tablename.c_str());
return false;
}
// get year
str.get(tmp, 5);
time = atoi(tmp);
if (time < 1970) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found a year < 1970 in \"%s\": %d", tablename.c_str(), time);
return false;
}
tableTime.tm_year = time - 1900;
// get month
str.get(tmp, 3);
time = atoi(tmp);
if (time < 1 || time > 12) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found a non-valid month in \"%s\": %d", tablename.c_str(), time);
return false;
}
tableTime.tm_mon = time - 1;
// get day
str.get(tmp, 3);
time = atoi(tmp);
if (time < 1 || time > 31) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found a non-valid day in \"%s\": %d", tablename.c_str(), time);
return false;
}
tableTime.tm_mday = time;
// get separamter "_"
str.get(tmp, 2);
if (std::string(tmp) != "_") {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found non-valid separator in \"%s\" after day: %s", tablename.c_str(), tmp);
return false;
}
// get hour
str.get(tmp, 3);
time = atoi(tmp);
if (time > 23) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found non-valid hour in \"%s\": %d", tablename.c_str(), time);
return false;
}
tableTime.tm_hour = time;
// get separator "_"
str.get(tmp, 2);
if (std::string(tmp) != "_") {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found non-valid separator in \"%s\" after hour: %s", tablename.c_str(), tmp);
return false;
}
// get half hour
str.get(tmp, 3);
time = atoi(tmp);
if (time == 0) {
// we are within the first half hour
tableTime.tm_min = 0;
} else if (time == 1) {
tableTime.tm_min = 30;
// we are in the second half hour
} else {
msg(MSG_ERROR, "IpfixDbReaderOracle: Found non-valid half hour indicator in \"%s\": %s", tablename.c_str(), tmp);
return false;
}
tableStartTime = timegm(&tableTime);
tableEndTime = tableStartTime + 30*60;
if (start) {
if (start > tableEndTime)
return false;
}
if (end) {
if (end < tableStartTime)
return false;
}
return true;
}
#endif

96
src/modules/ipfix/IpfixDbReaderOracle.hpp
View File

@ -1,96 +0,0 @@
/*
* IPFIX Database Reader/Writer for Oracle DBs
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006-2012 Lothar Braun <braun@net.in.tum.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef ORACLE_SUPPORT_ENABLED
#ifndef IPFIXDBREADERORACLE_H
#define IPFIXDBREADERORACLE_H
#include "IpfixRecord.hpp"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include "core/Module.h"
#include <netinet/in.h>
#include <time.h>
#include <pthread.h>
#include <boost/smart_ptr.hpp>
#include <occi.h>
/**
* IpfixDbReader powered the communication to the database server
* also between the other structs
*/
class IpfixDbReaderOracle : public Module, public Source<IpfixRecord*>, public Destination<NullEmitable*>
{
public:
IpfixDbReaderOracle(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint16_t observationDomainId,
bool timeshift, bool fullspeed, uint32_t firstFlow, uint32_t lastFlow);
~IpfixDbReaderOracle();
virtual void performStart();
virtual void performShutdown();
boost::shared_ptr<IpfixRecord::SourceID> srcId;
protected:
typedef struct {
uint16_t ipfixId; /**IPFIX_TYPEID*/
uint8_t length; /**IPFIX length*/
} columnDB;
vector<string> tables;
vector<columnDB> columns;
string columnNames;
string orderBy;
string whereClause;
unsigned recordLength;
bool timeshift, fullspeed;
uint32_t firstFlowTime, lastFlowTime;
bool dbError; // error flag
oracle::occi::Environment *env;
oracle::occi::Connection *con;
Thread thread;
static InstanceManager<IpfixTemplateRecord> templateRecordIM;
static InstanceManager<IpfixDataRecord> dataRecordIM;
static InstanceManager<IpfixTemplateDestructionRecord> templateDestructionRecordIM;
int getTables();
int getColumns(const string& tableName);
static void* readFromDB(void* ipfixDbReader_);
int dbReaderSendNewTemplate(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
int dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
int dbReaderDestroyTemplate(boost::shared_ptr<TemplateInfo> templateInfo);
int connectToDb( const string& hostName, const string& dbName, const string& userName, const string& password, unsigned int port);
bool isTableBetweenTimestamps(const string& tablename, uint32_t start, uint32_t end);
};
#endif
#endif

98
src/modules/ipfix/IpfixDbReaderOracleCfg.cpp
View File

@ -1,98 +0,0 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include "IpfixDbReaderOracleCfg.h"
IpfixDbReaderOracleCfg* IpfixDbReaderOracleCfg::create(XMLElement* e)
{
assert(e);
assert(e->getName() == getName());
return new IpfixDbReaderOracleCfg(e);
}
IpfixDbReaderOracleCfg::IpfixDbReaderOracleCfg(XMLElement* elem)
: CfgHelper<IpfixDbReaderOracle, IpfixDbReaderOracleCfg>(elem, "ipfixDbReaderOracle"),
port(0), timeshift(false), fullspeed(false), observationDomainId(0), startTime(0), endTime(0)
{
if (!elem) return;
XMLNode::XMLSet<XMLElement*> set = _elem->getElementChildren();
for (XMLNode::XMLSet<XMLElement*>::iterator it = set.begin();
it != set.end();
it++) {
XMLElement* e = *it;
if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
} else if (e->matches("dbname")) {
dbname = e->getFirstText();
} else if (e->matches("username")) {
user = e->getFirstText();
} else if (e->matches("password")) {
password = e->getFirstText();
} else if (e->matches("timeshift")) {
timeshift = getBool("timeshift", timeshift);
} else if (e->matches("fullspeed")) {
fullspeed = getBool("fullspeed", fullspeed);
} else if (e->matches("observationDomainId")) {
observationDomainId = getInt("observationDomainId");
} else if (e->matches("startTime")) {
startTime = getInt("startTime");
} else if (e->matches("endTime")) {
endTime = getInt("endTime");
} else if (e->matches("next")) { // ignore next
} else {
msg(MSG_FATAL, "Unknown IpfixDbReaderOracle config statement %s\n", e->getName().c_str());
continue;
}
}
if (hostname=="") THROWEXCEPTION("IpfixDbReaderOracleCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbReaderOracleCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbReaderOracleCfg: dbname not set in configuration!");
if (user=="") THROWEXCEPTION("IpfixDbReaderOracleCfg: username not set in configuration!");
if (password=="") THROWEXCEPTION("IpfixDbReaderOracleCfg: password not set in configuration!");
if (endTime != 0 && endTime < startTime) THROWEXCEPTION("IpfixDbReaderOracleCfg: endTime must be greater than startTime");
}
IpfixDbReaderOracleCfg::~IpfixDbReaderOracleCfg()
{
}
IpfixDbReaderOracle* IpfixDbReaderOracleCfg::createInstance()
{
instance = new IpfixDbReaderOracle(hostname, dbname, user, password, port, observationDomainId, timeshift, fullspeed, startTime, endTime);
return instance;
}
bool IpfixDbReaderOracleCfg::deriveFrom(IpfixDbReaderOracleCfg* old)
{
return false;
}
#endif /* ORACLE_SUPPORT_ENABLED */

64
src/modules/ipfix/IpfixDbReaderOracleCfg.h
View File

@ -1,64 +0,0 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifndef IPFIXDBREADERORACLECFG_H_
#define IPFIXDBREADERORACLECFG_H_
#ifdef ORACLE_SUPPORT_ENABLED
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "modules/ipfix/IpfixDbReaderOracle.hpp"
#include <string>
class IpfixDbReaderOracleCfg
: public CfgHelper<IpfixDbReaderOracle, IpfixDbReaderOracleCfg>
{
public:
friend class ConfigManager;
virtual IpfixDbReaderOracleCfg* create(XMLElement* e);
virtual ~IpfixDbReaderOracleCfg();
virtual IpfixDbReaderOracle* createInstance();
virtual bool deriveFrom(IpfixDbReaderOracleCfg* old);
protected:
string hostname; /**< hostname of database host */
uint16_t port; /**< port of database */
string dbname; /**< database name */
string user; /**< user name for login to database */
string password; /**< password for login to database */
bool timeshift; /**< shift time stamps */
bool fullspeed; /**< reading in full speed */
uint32_t observationDomainId; /**< observation domain id */
uint32_t startTime; /** FirstSwitchedTime of first flow that should be read from the DB */
uint32_t endTime; /** FirstSwitchedTime of last flow that should be read form the DB */
IpfixDbReaderOracleCfg(XMLElement*);
};
#endif /*ORACLE_SUPPORT_ENABLED*/
#endif /*IPFIXDBREADERORACLECFG_H_*/

646
src/modules/ipfix/IpfixDbWriter.cpp
View File

@ -1,646 +0,0 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007, 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef DB_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <iomanip>
#include <stdlib.h>
#include "IpfixDbWriter.hpp"
#include "common/msg.h"
/**
* struct to identify column names with IPFIX_TYPEID an the dataType to store in database
* ExporterID is no IPFIX_TYPEID, its user specified
* Attention: order of entries is important!
*/
const IpfixDbWriter::Column IpfixDbWriter::identify [] = {
{CN_dstIP, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_destinationIPv4Address, 0},
{CN_srcIP, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_sourceIPv4Address, 0},
{CN_srcPort, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_sourceTransportPort, 0},
{CN_dstPort, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_destinationTransportPort, 0},
{CN_proto, "TINYINT(3) UNSIGNED", 0, IPFIX_TYPEID_protocolIdentifier, 0 },
{CN_dstTos, "TINYINT(3) UNSIGNED", 0, IPFIX_TYPEID_classOfServiceIPv4, 0},
{CN_bytes, "BIGINT(20) UNSIGNED", 0, IPFIX_TYPEID_octetDeltaCount, 0},
{CN_pkts, "BIGINT(20) UNSIGNED", 0, IPFIX_TYPEID_packetDeltaCount, 0},
{CN_firstSwitched, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowStartSeconds, 0}, // default value is invalid/not used for this ent
{CN_lastSwitched, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowEndSeconds, 0}, // default value is invalid/not used for this entry
{CN_firstSwitchedMillis, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_flowStartMilliSeconds, 0},
{CN_lastSwitchedMillis, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_flowEndMilliSeconds, 0},
{CN_tcpControlBits, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_tcpControlBits, 0},
//TODO: use enterprise number for the following extended types (Gerhard, 12/2009)
{CN_revbytes, "BIGINT(20) UNSIGNED", 0, IPFIX_TYPEID_octetDeltaCount, IPFIX_PEN_reverse},
{CN_revpkts, "BIGINT(20) UNSIGNED", 0, IPFIX_TYPEID_packetDeltaCount, IPFIX_PEN_reverse},
{CN_revFirstSwitched, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowStartSeconds, IPFIX_PEN_reverse}, // default value is invalid/not used for this entry
{CN_revLastSwitched, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowEndSeconds, IPFIX_PEN_reverse}, // default value is invalid/not used for this entry
{CN_revFirstSwitchedMillis, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_flowStartMilliSeconds, IPFIX_PEN_reverse},
{CN_revLastSwitchedMillis, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_flowEndMilliSeconds, IPFIX_PEN_reverse},
{CN_revTcpControlBits, "SMALLINT(5) UNSIGNED", 0, IPFIX_TYPEID_tcpControlBits, IPFIX_PEN_reverse},
{CN_maxPacketGap, "BIGINT(20) UNSIGNED", 0, IPFIX_ETYPEID_maxPacketGap, IPFIX_PEN_vermont|IPFIX_PEN_reverse},
{CN_exporterID, "SMALLINT(5) UNSIGNED", 0, EXPORTERID, 0},
{CN_flowStartSysUpTime, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowStartSysUpTime, 0},
{CN_flowEndSysUpTime, "INTEGER(10) UNSIGNED", 0, IPFIX_TYPEID_flowEndSysUpTime, 0},
{0} // last entry must be 0
};
/**
* Compare two source IDs and check if exporter is the same (i.e., same IP address and observationDomainId
*/
bool IpfixDbWriter::equalExporter(const IpfixRecord::SourceID& a, const IpfixRecord::SourceID& b) {
return (a.observationDomainId == b.observationDomainId) &&
(a.exporterAddress.len == b.exporterAddress.len) &&
(memcmp(a.exporterAddress.ip, b.exporterAddress.ip, a.exporterAddress.len) == 0 );
}
/**
* (re)connect to database
*/
int IpfixDbWriter::connectToDB()
{
ostringstream statement;
dbError = true;
// close (in the case that it was already connected)
if (conn) mysql_close(conn);
/** get the mysl init handle*/
conn = mysql_init(0);
if(conn == 0) {
msg(MSG_FATAL,"IpfixDbWriter: Get MySQL connect handle failed. Error: %s",
mysql_error(conn));
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriter: mysql init successful");
/**Connect to Database*/
if (!mysql_real_connect(conn, dbHost.c_str(), dbUser.c_str(), dbPassword.c_str(),
0, dbPort, 0, 0)) {
msg(MSG_FATAL,"IpfixDbWriter: Connection to database failed. Error: %s",
mysql_error(conn));
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriter: succesfully connected to database");
/** make query string to create database**/
statement << "CREATE DATABASE IF NOT EXISTS " << dbName;
DPRINTF("SQL Query: %s", statement.str().c_str());
/**create database*/
if(mysql_query(conn, statement.str().c_str()) != 0 ) {
msg(MSG_FATAL, "IpfixDbWriter: Creation of database %s failed. Error: %s",
dbName.c_str(), mysql_error(conn));
return 1;
}
msg(MSG_INFO,"IpfixDbWriter: Database %s created", dbName.c_str());
/** use database with dbName**/
if(mysql_select_db(conn, dbName.c_str()) !=0) {
msg(MSG_FATAL, "IpfixDbWriter: Database %s not selectable. Error: %s",
dbName.c_str(), mysql_error(conn));
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriter: Database %s selected", dbName.c_str());
/**create table exporter*/
statement.str("");
statement.clear();
statement << "CREATE TABLE IF NOT EXISTS exporter (id SMALLINT(5) NOT NULL AUTO_INCREMENT, sourceID INTEGER(10) UNSIGNED DEFAULT NULL, srcIP INTEGER(10) UNSIGNED DEFAULT NULL, PRIMARY KEY(id))";
DPRINTF("SQL Query: %s", statement.str().c_str());
if(mysql_query(conn, statement.str().c_str()) != 0) {
msg(MSG_FATAL,"IpfixDbWriter: Creation of exporter table failed. Error: %s",
mysql_error(conn));
return 1;
}
msg(MSG_INFO,"IpfixDbWriter: Exporter table created");
dbError = false;
return 0;
}
/**
* save record to database
*/
void IpfixDbWriter::processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data)
{
string rowString;
time_t flowStartSeconds;
DPRINTF("Processing data record");
if (dbError) {
connectToDB();
if (dbError) return;
}
/* get new insert */
if(srcId.observationDomainId != 0) {
// use default source id
rowString = getInsertString(rowString, flowStartSeconds, srcId, dataTemplateInfo, length, data);
} else {
rowString = getInsertString(rowString, flowStartSeconds, sourceID, dataTemplateInfo, length, data);
}
// if current table is not ok, write to db and get new table name
if(!(flowStartSeconds >= currentTable.startTime && flowStartSeconds <= currentTable.endTime)) {
if(numberOfInserts > 0) {
msg(MSG_DEBUG, "IpfixDbWriter: Writing buffered records to database");
writeToDb();
numberOfInserts = 0;
}
if (setCurrentTable(flowStartSeconds) != 0) {
return;
}
}
// start new insert statement if necessary
if (numberOfInserts == 0) {
// start insert statement
insertStatement.str("");
insertStatement.clear();
insertStatement << "INSERT INTO " << currentTable.name << " (" << tableColumnsString << ") VALUES " << rowString;
numberOfInserts = 1;
} else {
// append insert statement
insertStatement << "," << rowString;
numberOfInserts++;
}
// write to db if maxInserts is reached
if(numberOfInserts == maxInserts) {
msg(MSG_DEBUG, "IpfixDbWriter: Writing buffered records to database");
writeToDb();
numberOfInserts = 0;
}
}
/**
* loop over table columns and template to get the IPFIX values in correct order to store in database
* The result is written into row, the firstSwitched time is returned in flowstartsec
*/
string& IpfixDbWriter::getInsertString(string& row, time_t& flowstartsec, const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data)
{
uint64_t intdata = 0;
uint64_t intdata2 = 0;
uint32_t k;
bool notfound, notfound2;
bool first = true;
ostringstream rowStream(row);
flowstartsec = 0;
rowStream << "(";
/**loop over the columname and loop over the IPFIX_TYPEID of the record
to get the corresponding data to store and make insert statement*/
for(vector<Column>::iterator col = tableColumns.begin(); col != tableColumns.end(); col++) {
if (col->ipfixId == EXPORTERID) {
// if this is the same source ID as last time, we get the exporter id from currentExporter
if ((currentExporter != NULL) && equalExporter(sourceID, currentExporter->sourceID)) {
DPRINTF("Exporter is same as last time (ODID=%d, id=%d)", sourceID.observationDomainId, currentExporter->id);
intdata = (uint64_t)currentExporter->id;
} else {
// lookup exporter buffer to get exporterID from sourcID and expIp
intdata = (uint64_t)getExporterID(sourceID);
}
} else {
notfound = true;
// try to gather data required for the field
if(dataTemplateInfo.fieldCount > 0) {
// look inside the ipfix record
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo.fieldInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
DPRINTF("IpfixDbWriter::getData: really saw ipfix id %d in packet with intdata %llX, type %d, length %d and offset %X", col->ipfixId, intdata, dataTemplateInfo.fieldInfo[k].type.id, dataTemplateInfo.fieldInfo[k].type.length, dataTemplateInfo.fieldInfo[k].offset);
break;
}
}
}
if( dataTemplateInfo.dataCount > 0 && notfound) {
// look in static data fields of template for data
for(k=0; k < dataTemplateInfo.dataCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo.dataInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getData(dataTemplateInfo.dataInfo[k].type,(dataTemplateInfo.data+dataTemplateInfo.dataInfo[k].offset));
break;
}
}
}
if(notfound) {
notfound2 = true;
// for some Ids, we have an alternative
if(col->enterprise == 0) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
// look for alternative (flowStartMilliSeconds/1000)
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowStartMilliSeconds) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow start time is available, maybe this is is from a netflow from Cisco
// then - as a last alternative - use flowStartSysUpTime as flow start time
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSysUpTime) {
intdata2 = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
// look for alternative (flowEndMilliSeconds/1000)
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowEndMilliSeconds) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow end time is available, maybe this is is from a netflow from Cisco
// then use flowEndSysUpTime as flow start time
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSysUpTime) {
intdata2 = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// look for alternative (revFlowStartMilliSeconds/1000)
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
// look for alternative (revFlowEndMilliSeconds/1000)
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
}
}
// if still not found, get default value
if(notfound)
intdata = col->defaultValue;
}
// we need extra treatment for timing related fields
if(col->enterprise == 0 ) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// save time for table access
if (flowstartsec==0) flowstartsec = intdata;
break;
case IPFIX_TYPEID_flowEndSeconds:
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
// if flowStartSeconds is not stored in one of the columns, but flowStartMilliSeconds is,
// then we use flowStartMilliSeconds for table access
// This is realized by storing this value only if flowStartSeconds has not yet been seen.
// A later appearing flowStartSeconds will override this value.
if (flowstartsec==0)
flowstartsec = intdata/1000;
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
intdata %= 1000;
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse)
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
intdata %= 1000;
break;
}
}
DPRINTF("saw ipfix id %d in packet with intdata %llX", col->ipfixId, intdata);
if(first)
rowStream << intdata;
else
rowStream << "," << intdata;
first = false;
}
rowStream << ")";
if (flowstartsec == 0) {
msg(MSG_ERROR, "IpfixDbWriter: Failed to get timing data from record. Will be saved in default table.");
}
row = rowStream.str();
DPRINTF("Insert row: %s", row.c_str());
return row;
}
/*
* Write insertStatement to database
*/
int IpfixDbWriter::writeToDb()
{
DPRINTF("SQL Query: %s", insertStatement.str().c_str());
if(mysql_query(conn, insertStatement.str().c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbWriter: Insert of records failed. Error: %s", mysql_error(conn));
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriter: Write to database is complete");
return 0;
}
/*
* Sets the current table information and creates the table in the database if necessary
*/
int IpfixDbWriter::setCurrentTable(time_t flowstartsec)
{
// generate table name
ostringstream tableStream;
struct tm* flowStartTime = gmtime(&flowstartsec);
tableStream << "h_" << (flowStartTime->tm_year+1900)
<< setfill('0') << setw(2) << (flowStartTime->tm_mon+1)
<< setfill('0') << setw(2) << (flowStartTime->tm_mday) << "_"
<< setfill('0') << setw(2) << (flowStartTime->tm_hour) << "_"
<< setw(1) << (flowStartTime->tm_min<30?0:1);
currentTable.name = tableStream.str();
// calculate table boundaries
if(flowStartTime->tm_min < 30) {
flowStartTime->tm_min = 0;
flowStartTime->tm_sec = 0;
currentTable.startTime = timegm(flowStartTime);
} else {
flowStartTime->tm_min = 30;
flowStartTime->tm_sec = 0;
currentTable.startTime = timegm(flowStartTime);
}
currentTable.endTime = currentTable.startTime + 1799;
DPRINTF("flowstartsec: %d, table name: %s, start time: %d, end time: %d", flowstartsec, currentTable.name.c_str(), currentTable.startTime, currentTable.endTime);
// create table if exists
ostringstream createStatement;
createStatement << "CREATE TABLE IF NOT EXISTS " << currentTable.name << " (" << tableColumnsCreateString << ")";
DPRINTF("SQL Query: %s", createStatement.str().c_str());
if(mysql_query(conn, createStatement.str().c_str()) != 0) {
msg(MSG_FATAL,"IpfixDbWriter: Creation of table failed. Error: %s", mysql_error(conn));
dbError = true;
return 1;
}
msg(MSG_DEBUG, "IpfixDbWriter: Table %s created ", currentTable.name.c_str());
return 0;
}
/**
* Returns the id of the exporter table entry or 0 in the case of an error
*/
int IpfixDbWriter::getExporterID(const IpfixRecord::SourceID& sourceID)
{
list<ExporterCacheEntry>::iterator iter;
MYSQL_RES* dbResult;
MYSQL_ROW dbRow;
int id = 0;
uint32_t expIp = 0;
ostringstream statement;
iter = exporterCache.begin();
while(iter != exporterCache.end()) {
if (equalExporter(iter->sourceID, sourceID)) {
// found exporter in exporterCache
DPRINTF("Exporter (ODID=%d, id=%d) found in exporter cache", sourceID.observationDomainId, iter->id);
exporterCache.push_front(*iter);
exporterCache.erase(iter);
// update current exporter
currentExporter = &exporterCache.front();
return exporterCache.front().id;
}
iter++;
}
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID.exporterAddress.toUInt32();
// search exporter table
statement << "SELECT id FROM exporter WHERE sourceID=" << sourceID.observationDomainId << " AND srcIp=" << expIp;
DPRINTF("SQL Query: %s", statement.str().c_str());
if(mysql_query(conn, statement.str().c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbWriter: Select on exporter table failed. Error: %s",
mysql_error(conn));
return 0;// If a failure occurs, return 0
}
dbResult = mysql_store_result(conn);
if(( dbRow = mysql_fetch_row(dbResult))) {
// found in table
id = atoi(dbRow[0]);
mysql_free_result(dbResult);
DPRINTF("ExporterID %d is in exporter table", id);
} else {
mysql_free_result(dbResult);
// insert new exporter table entry
statement.str("");
statement.clear();
statement << "INSERT INTO exporter (ID,sourceID,srcIP) VALUES ('NULL','" << sourceID.observationDomainId << "','" << expIp << "')";
DPRINTF("SQL Query: %s", statement.str().c_str());
if(mysql_query(conn, statement.str().c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbWriter: Insert in exporter table failed. Error: %s", conn);
return 0;
}
id = mysql_insert_id(conn);
msg(MSG_INFO,"IpfixDbWriter: new exporter (ODID=%d, id=%d) inserted in exporter table", sourceID.observationDomainId, id);
}
// insert exporter in cache
ExporterCacheEntry tmp = {sourceID, id};
exporterCache.push_front(tmp);
// update current exporter
currentExporter = &exporterCache.front();
// pop last element if exporter cache is to long
if(exporterCache.size() > MAX_EXPORTER)
exporterCache.pop_back();
return id;
}
/**
* Get data of the record is given by the IPFIX_TYPEID
*/
uint64_t IpfixDbWriter::getData(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
switch (type.length) {
case 1:
return (*(uint8_t*)data);
case 2:
return ntohs(*(uint16_t*)data);
case 4:
return ntohl(*(uint32_t*)data);
case 5: // may occur in the case if IP address + mask
return ntohl(*(uint32_t*)data);
case 8:
return ntohll(*(uint64_t*)data);
default:
printf("Uint with length %d unparseable", type.length);
return 0;
}
}
/***** Public Methods ****************************************************/
/**
* called on Data Record arrival
*/
void IpfixDbWriter::onDataRecord(IpfixDataRecord* record)
{
// only treat non-Options Data Records (although we cannot be sure that there is a Flow inside)
if((record->templateInfo->setId != TemplateInfo::NetflowTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixDataTemplate)) {
record->removeReference();
return;
}
processDataDataRecord(*record->sourceID.get(), *record->templateInfo.get(),
record->dataLength, record->data);
record->removeReference();
}
/**
* Constructor
*/
IpfixDbWriter::IpfixDbWriter(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint32_t observationDomainId, unsigned maxStatements,
const vector<string>& columns)
: currentExporter(NULL), numberOfInserts(0), maxInserts(maxStatements),
dbHost(hostname), dbName(dbname), dbUser(username), dbPassword(password), dbPort(port), conn(0)
{
int i;
// set default source id
srcId.exporterAddress.len = 0;
srcId.observationDomainId = observationDomainId;
srcId.exporterPort = 0;
srcId.receiverPort = 0;
srcId.protocol = 0;
srcId.fileDescriptor = 0;
// invalide start settings for current table (to enforce table create)
currentTable.startTime = 1;
currentTable.endTime = 0;
if(columns.empty())
THROWEXCEPTION("IpfixDbWriter: cannot initiate with no columns");
/* get columns */
bool first = true;
for(vector<string>::const_iterator col = columns.begin(); col != columns.end(); col++) {
i = 0;
while(identify[i].columnName != 0) {
if(col->compare(identify[i].columnName) == 0) {
Column c = identify[i];
tableColumns.push_back(c);
// update tableColumnsString
if(!first)
tableColumnsString.append(",");
tableColumnsString.append(identify[i].columnName);
// update tableColumnsCreateString
if(!first)
tableColumnsCreateString.append(", ");
tableColumnsCreateString.append(identify[i].columnName);
tableColumnsCreateString.append(" ");
tableColumnsCreateString.append(identify[i].columnType);
first = false;
break;
}
i++;
}
}
msg(MSG_INFO, "IpfixDbWriter: columns are %s", tableColumnsString.c_str());
if(connectToDB() != 0)
THROWEXCEPTION("IpfixDbWriter creation failed");
}
/**
* Destructor
*/
IpfixDbWriter::~IpfixDbWriter()
{
writeToDb();
mysql_close(conn);
}
#endif

132
src/modules/ipfix/IpfixDbWriter.hpp
View File

@ -1,132 +0,0 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007, 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef DB_SUPPORT_ENABLED
#ifndef IPFIXDBWRITER_H
#define IPFIXDBWRITER_H
#include "IpfixDbCommon.hpp"
#include "IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <mysql.h>
#include <netinet/in.h>
#include <time.h>
#include <sstream>
#define EXPORTERID 0
/**
* IpfixDbWriter powered the communication to the database server
* also between the other structs
*/
class IpfixDbWriter
: public IpfixRecordDestination, public Module, public Source<NullEmitable*>
{
public:
IpfixDbWriter(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint32_t observationDomainId, unsigned maxStatements,
const vector<string>& columns);
~IpfixDbWriter();
void onDataRecord(IpfixDataRecord* record);
/**
* Struct to identify the relationship between columns names and
* IPFIX_TYPEID, column type and default value
*/
struct Column {
const char* columnName; /** column name */
const char* columnType; /** column data type in database */
uint64_t defaultValue; /** default value */
InformationElement::IeId ipfixId; /** IPFIX_TYPEID */
InformationElement::IeEnterpriseNumber enterprise; /** enterprise number */
};
private:
static const unsigned MAX_EXPORTER = 10; // maximum numbers of cached exporters
/**
* Struct buffers start and end time and tablename for the different tables
*/
struct TableCacheEntry {
time_t startTime; // smallest flow start second timestamp in the table
time_t endTime; // largest flow start second timestamp in the table
string name; // name of the table
};
/**
* Struct buffers ODID, IP address and row index of an exporter
*/
struct ExporterCacheEntry {
IpfixRecord::SourceID sourceID;/** source id of the exporter */
int id; /** Id entry of sourcID and expIP in the ExporterTable */
};
TableCacheEntry currentTable; // current table in tableCache
list<ExporterCacheEntry> exporterCache; // cached tables names, key=observationDomainId
ExporterCacheEntry* currentExporter; // pointer to current exporter in exporterCache
IpfixRecord::SourceID srcId; // default source ID
ostringstream insertStatement; // insert statement string
int numberOfInserts; // number of inserts in statement
int maxInserts; // maximum number of inserts per statement
vector<Column> tableColumns; // table columns
string tableColumnsString; // table columns as string for INSERT statements
string tableColumnsCreateString; // table columns as string for CREATE statements
// database data
string dbHost, dbName, dbUser, dbPassword;
unsigned dbPort;
MYSQL* conn; /** pointer to connection handle */
bool dbError; // db error flag
int createDB();
int setCurrentTable(time_t flowstartsec);
string& getInsertString(string& row, time_t& flowstartsec, const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data);
int writeToDb();
int getExporterID(const IpfixRecord::SourceID& sourceID);
int connectToDB();
void processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data);
uint64_t getData(InformationElement::IeInfo type, IpfixRecord::Data* data);
bool equalExporter(const IpfixRecord::SourceID& a, const IpfixRecord::SourceID& b);
const static Column identify[];
};
#endif
#endif

917
src/modules/ipfix/IpfixDbWriterOracle.cpp
View File

@ -1,917 +0,0 @@
/*
* IPFIX Database Reader/Writer Oracle Connector
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <iomanip>
#include <stdlib.h>
#include "IpfixDbWriterOracle.hpp"
#include "common/msg.h"
const IpfixDbWriterOracle::Column IpfixDbWriterOracle::identify [] = {
{CN_dstIP, "NUMBER(10)", 0, IPFIX_TYPEID_destinationIPv4Address, 0},
{CN_srcIP, "NUMBER(10)", 0, IPFIX_TYPEID_sourceIPv4Address, 0},
{CN_srcPort, "NUMBER(5)", 0, IPFIX_TYPEID_sourceTransportPort, 0},
{CN_dstPort, "NUMBER(5)", 0, IPFIX_TYPEID_destinationTransportPort, 0},
{CN_proto, "NUMBER(3)", 0, IPFIX_TYPEID_protocolIdentifier, 0 },
{CN_dstTos, "NUMBER(3)", 0, IPFIX_TYPEID_classOfServiceIPv4, 0},
{CN_bytes, "NUMBER(20)", 0, IPFIX_TYPEID_octetDeltaCount, 0},
{CN_pkts, "NUMBER(20)", 0, IPFIX_TYPEID_packetDeltaCount, 0},
{CN_firstSwitched, "NUMBER(10)", 0, IPFIX_TYPEID_flowStartSeconds, 0}, // default value is invalid/not used for this ent
{CN_lastSwitched, "NUMBER(10)", 0, IPFIX_TYPEID_flowEndSeconds, 0}, // default value is invalid/not used for this entry
{CN_firstSwitchedMillis, "NUMBER(5)", 0, IPFIX_TYPEID_flowStartMilliSeconds, 0},
{CN_lastSwitchedMillis, "NUMBER(5)", 0, IPFIX_TYPEID_flowEndMilliSeconds, 0},
{CN_tcpControlBits, "NUMBER(5)", 0, IPFIX_TYPEID_tcpControlBits, 0},
//TODO: use enterprise number for the following extended types (Gerhard, 12/2009)
{CN_revbytes, "NUMBER(20)", 0, IPFIX_TYPEID_octetDeltaCount, IPFIX_PEN_reverse},
{CN_revpkts, "NUMBER(20)", 0, IPFIX_TYPEID_packetDeltaCount, IPFIX_PEN_reverse},
{CN_revFirstSwitched, "NUMBER(10)", 0, IPFIX_TYPEID_flowStartSeconds, IPFIX_PEN_reverse}, // default value is invalid/not used for this entry
{CN_revLastSwitched, "NUMBER(10)", 0, IPFIX_TYPEID_flowEndSeconds, IPFIX_PEN_reverse}, // default value is invalid/not used for this entry
{CN_revFirstSwitchedMillis, "NUMBER(5)", 0, IPFIX_TYPEID_flowStartMilliSeconds, IPFIX_PEN_reverse},
{CN_revLastSwitchedMillis, "NUMBER(5)", 0, IPFIX_TYPEID_flowEndMilliSeconds, IPFIX_PEN_reverse},
{CN_revTcpControlBits, "NUMBER(5)", 0, IPFIX_TYPEID_tcpControlBits, IPFIX_PEN_reverse},
{CN_maxPacketGap, "NUMBER(20)", 0, IPFIX_ETYPEID_maxPacketGap, IPFIX_PEN_vermont|IPFIX_PEN_reverse},
{CN_exporterID, "NUMBER(5)", 0, EXPORTERID, 0},
{0} // last entry must be 0
};
/**
* Compare two source IDs and check if exporter is the same (i.e., same IP address and observationDomainId
*/
bool IpfixDbWriterOracle::equalExporter(const IpfixRecord::SourceID& a, const IpfixRecord::SourceID& b) {
return (a.observationDomainId == b.observationDomainId) &&
(a.exporterAddress.len == b.exporterAddress.len) &&
(memcmp(a.exporterAddress.ip, b.exporterAddress.ip, a.exporterAddress.len) == 0 );
}
/**
* (re)connect to database
*/
int IpfixDbWriterOracle::connectToDB()
{
dbError = true;
// close (in the case that it was already connected)
if (con) env->terminateConnection(con);
/** get the initial environment and connect */
msg(MSG_DEBUG, "IpfixDbWriterOracle: Creating environment.");
try {
env = oracle::occi::Environment::createEnvironment(oracle::occi::Environment::DEFAULT);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbWriterOracle: Error while creating environment: %s.", ex.getMessage().c_str());
msg(MSG_FATAL, "IpfixDbWriterOracle: Did you configure your Oracle environment?");
return -1;
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Trying to connect to database ...");
try
{
char dbLogon[256];
sprintf(dbLogon, "%s:%u/%s", dbHost.c_str(), dbPort, dbName.c_str());
con = env->createConnection(dbUser, dbPassword, dbLogon);
} catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Oracle connect failed. Error: %s", ex.getMessage().c_str());
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: Oracle connection successful");
if (createExporterTable()!=0) return 1;
dbError = false;
return 0;
}
int IpfixDbWriterOracle::createExporterTable()
{
// check if table exists
ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT COUNT(table_name) FROM user_tables WHERE table_name='EXPORTER'";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 1;
}
if (rs)
{
while(rs->next())
{
if (rs->getInt(1)!= 0)
{
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table does exist");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
return 0;
}
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
// create table
sql.str("");
sql << "CREATE TABLE exporter ( id NUMERIC(10) NOT NULL, sourceID NUMERIC(10), srcIP NUMERIC(10), CONSTRAINT exporter_pk PRIMARY KEY (id) )";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating exporter table statement: %s", ex.getMessage().c_str());
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating exporter table: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
// create counter
// clear vars for reuse
sql.str("");
sql << "CREATE sequence counter_for_exporter increment BY 1 start WITH 1 cache 2";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating sequence counter statement: %s", ex.getMessage().c_str());
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating squence counter table: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table counter created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
// create trigger
sql.str("");
sql << "CREATE OR REPLACE TRIGGER trigger_for_id_exporter BEFORE INSERT ON exporter REFERENCING NEW AS new FOR EACH ROW Begin SELECT counter_for_exporter.NEXTVAL INTO :new.id FROM DUAL; End;";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table insert trigger created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
msg(MSG_DEBUG, "Exporter table creation done");
return 0;
}
/**
* save record to database
*/
void IpfixDbWriterOracle::processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data)
{
string rowString;
time_t flowStartSeconds;
msg(MSG_DEBUG, "IpfixDbWriterOracle: Processing data record");
if (dbError) {
msg(MSG_DEBUG, "IpfixDbWriterOracle: reconnecting to DB");
connectToDB();
if (dbError) return;
}
/* get new insert */
if(srcId.observationDomainId != 0) {
// use default source id
rowString = getInsertString(rowString, flowStartSeconds, srcId, dataTemplateInfo, length, data);
} else {
rowString = getInsertString(rowString, flowStartSeconds, sourceID, dataTemplateInfo, length, data);
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Row: %s", rowString.c_str());
// if current table is not ok, write to db and get new table name
if(!(flowStartSeconds >= currentTable.startTime && flowStartSeconds <= currentTable.endTime)) {
if(numberOfInserts > 0) {
msg(MSG_DEBUG, "IpfixDbWriterOracle: Writing buffered records to database");
insertStatement << " SELECT * FROM dual";
writeToDb();
numberOfInserts = 0;
}
if (setCurrentTable(flowStartSeconds) != 0) {
return;
}
}
// start new insert statement if necessary
if (numberOfInserts == 0) {
// start insert statement
insertStatement.str("");
insertStatement.clear();
insertStatement << "INSERT ALL INTO " << currentTable.name << " (" << tableColumnsString << ") VALUES " << rowString;
numberOfInserts = 1;
} else {
// append insert statement
insertStatement << " INTO " << currentTable.name << " (" << tableColumnsString << ") VALUES " << rowString;
numberOfInserts++;
}
// write to db if maxInserts is reached
if(numberOfInserts == maxInserts) {
msg(MSG_DEBUG, "IpfixDbWriterOracle: Writing buffered records to database");
insertStatement << " SELECT * FROM dual";
writeToDb();
numberOfInserts = 0;
}
}
/**
* loop over table columns and template to get the IPFIX values in correct order to store in database
* The result is written into row, the firstSwitched time is returned in flowstartsec
*/
string& IpfixDbWriterOracle::getInsertString(string& row, time_t& flowstartsec, const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data)
{
uint64_t intdata = 0;
uint64_t intdata2 = 0;
uint32_t k;
bool notfound, notfound2;
bool first = true;
ostringstream rowStream(row);
flowstartsec = 0;
rowStream << "(";
/**loop over the columname and loop over the IPFIX_TYPEID of the record
to get the corresponding data to store and make insert statement*/
for(vector<Column>::iterator col = tableColumns.begin(); col != tableColumns.end(); col++) {
if (col->ipfixId == EXPORTERID) {
// if this is the same source ID as last time, we get the exporter id from currentExporter
if ((currentExporter != NULL) && equalExporter(sourceID, currentExporter->sourceID)) {
DPRINTF("Exporter is same as last time (ODID=%d, id=%d)", sourceID.observationDomainId, currentExporter->id);
intdata = (uint64_t)currentExporter->id;
} else {
// lookup exporter buffer to get exporterID from sourcID and expIp
intdata = (uint64_t)getExporterID(sourceID);
}
} else {
notfound = true;
// try to gather data required for the field
if(dataTemplateInfo.fieldCount > 0) {
// look inside the ipfix record
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo.fieldInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
DPRINTF("IpfixDbWriterOracle::getData: really saw ipfix id %d in packet with intdata %llX, type %d, length %d and offset %X", col->ipfixId, intdata, dataTemplateInfo.fieldInfo[k].type.id, dataTemplateInfo.fieldInfo[k].type.length, dataTemplateInfo.fieldInfo[k].offset);
break;
}
}
}
if( dataTemplateInfo.dataCount > 0 && notfound) {
// look in static data fields of template for data
for(k=0; k < dataTemplateInfo.dataCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo.dataInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getData(dataTemplateInfo.dataInfo[k].type,(dataTemplateInfo.data+dataTemplateInfo.dataInfo[k].offset));
break;
}
}
}
if(notfound) {
notfound2 = true;
// for some Ids, we have an alternative
if(col->enterprise == 0) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
// look for alternative (flowStartMilliSeconds/1000)
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowStartMilliSeconds) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow start time is available, maybe this is is from a netflow from Cisco
// then - as a last alternative - use flowStartSysUpTime as flow start time
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSysUpTime) {
intdata2 = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
// look for alternative (flowEndMilliSeconds/1000)
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowEndMilliSeconds) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow end time is available, maybe this is is from a netflow from Cisco
// then use flowEndSysUpTime as flow start time
if(dataTemplateInfo.fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSysUpTime) {
intdata2 = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// look for alternative (revFlowStartMilliSeconds/1000)
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
// look for alternative (revFlowEndMilliSeconds/1000)
if(dataTemplateInfo.fieldCount > 0) {
for(k=0; k < dataTemplateInfo.fieldCount; k++) {
if(dataTemplateInfo.fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
}
}
// if still not found, get default value
if(notfound)
intdata = col->defaultValue;
}
// we need extra treatment for timing related fields
if(col->enterprise == 0 ) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// save time for table access
if (flowstartsec==0) flowstartsec = intdata;
break;
case IPFIX_TYPEID_flowEndSeconds:
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
// if flowStartSeconds is not stored in one of the columns, but flowStartMilliSeconds is,
// then we use flowStartMilliSeconds for table access
// This is realized by storing this value only if flowStartSeconds has not yet been seen.
// A later appearing flowStartSeconds will override this value.
if (flowstartsec==0)
flowstartsec = intdata/1000;
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
intdata %= 1000;
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse)
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
intdata %= 1000;
break;
}
}
DPRINTF("saw ipfix id %d in packet with intdata %llX", col->ipfixId, intdata);
if(first)
rowStream << intdata;
else
rowStream << "," << intdata;
first = false;
}
rowStream << ")";
if (flowstartsec == 0) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Failed to get timing data from record. Will be saved in default table.");
}
row = rowStream.str();
DPRINTF("Insert row: %s", row.c_str());
return row;
}
/*
* Write insertStatement to database
*/
int IpfixDbWriterOracle::writeToDb()
{
//msg(MSG_DEBUG, "SQL Query: %s", insertStatement.str().c_str());
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
try
{
stmt = con->createStatement(insertStatement.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 1;
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
msg(MSG_DEBUG,"IpfixDbWriterOracle: Write to database is complete");
return 0;
}
return 1;
}
/*
* Sets the current table information and creates the table in the database if necessary
*/
int IpfixDbWriterOracle::setCurrentTable(time_t flowstartsec)
{
// generate table name
ostringstream tableStream;
struct tm* flowStartTime = gmtime(&flowstartsec);
tableStream << "H_" << (flowStartTime->tm_year+1900)
<< setfill('0') << setw(2) << (flowStartTime->tm_mon+1)
<< setfill('0') << setw(2) << (flowStartTime->tm_mday) << "_"
<< setfill('0') << setw(2) << (flowStartTime->tm_hour) << "_"
<< setw(1) << (flowStartTime->tm_min<30?0:1);
currentTable.name = tableStream.str();
// calculate table boundaries
if(flowStartTime->tm_min < 30) {
flowStartTime->tm_min = 0;
flowStartTime->tm_sec = 0;
currentTable.startTime = timegm(flowStartTime);
} else {
flowStartTime->tm_min = 30;
flowStartTime->tm_sec = 0;
currentTable.startTime = timegm(flowStartTime);
}
currentTable.endTime = currentTable.startTime + 1799;
msg(MSG_DEBUG, "IpfixDbWriterOracle: flowstartsec: %d, table name: %s, start time: %d, end time: %d", flowstartsec, currentTable.name.c_str(), currentTable.startTime, currentTable.endTime);
// check if table exists
ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT COUNT(table_name) FROM user_tables WHERE table_name='" << currentTable.name << "'";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
if (rs)
{
while(rs->next())
{
if (rs->getInt(1)!= 0)
{
msg(MSG_DEBUG,"IpfixDbWriterOracle: table does exist");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
return 0;
}
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
// create table
sql.str("");
sql << "CREATE TABLE " << currentTable.name << " ( " << tableColumnsCreateString << ")";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Table %s created ", currentTable.name.c_str());
return 0;
}
/**
* Returns the id of the exporter table entry or 0 in the case of an error
*/
int IpfixDbWriterOracle::getExporterID(const IpfixRecord::SourceID& sourceID)
{
list<ExporterCacheEntry>::iterator iter;
oracle::occi::Statement* stmt = NULL;
oracle::occi::ResultSet* rs = NULL;
int id = -1;
uint32_t expIp = 0;
ostringstream sql;
iter = exporterCache.begin();
while(iter != exporterCache.end()) {
if (equalExporter(iter->sourceID, sourceID)) {
// found exporter in exporterCache
DPRINTF("Exporter (ODID=%d, id=%d) found in exporter cache", sourceID.observationDomainId, iter->id);
exporterCache.push_front(*iter);
exporterCache.erase(iter);
// update current exporter
currentExporter = &exporterCache.front();
return exporterCache.front().id;
}
iter++;
}
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID.exporterAddress.toUInt32();
// search exporter table
sql << "SELECT id FROM exporter WHERE sourceID=" << sourceID.observationDomainId << " AND srcIp=" << expIp;
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on exporter table failed. Error: %s", ex.getMessage().c_str());
return 0;// If a failure occurs, return 0
}
if(stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
if (rs)
{
while(rs->next())
{
id = rs->getInt(1);
msg(MSG_DEBUG, "IpfixDbWriterOracle: ExporterID %d is in exporter table", id);
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on exporter table failed. Error: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 0;// If a failure occurs, return 0
}
}
// insert new entry in exporter table since it is not found
if(id == -1)
{
sql.str("");
sql << "INSERT INTO exporter (sourceID,srcIP) VALUES ('" << sourceID.observationDomainId << "','" << expIp << "')";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Insert in exporter table failed. Error: %s", ex.getMessage().c_str());
return 0;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Insert in exporter table failed. Error: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 0;
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
sql.str("");
sql << "SELECT counter_for_exporter.CURRVAL FROM DUAL";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on counter_for_exporter sequence failed. Error: %s", ex.getMessage().c_str());
return 0;// If a failure occurs, return 0
}
if(stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
if (rs)
{
while(rs->next())
{
id = rs->getInt(1);
DPRINTF("ExporterID %d is in exporter table", id);
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on counter_for_exporter sequence failed. Error: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
return 0;// If a failure occurs, return 0
}
msg(MSG_INFO,"IpfixDbWriterOracle: new exporter (ODID=%d, id=%d) inserted in exporter table", sourceID.observationDomainId, id);
}
}
// insert exporter in cache
ExporterCacheEntry tmp = {sourceID, id};
exporterCache.push_front(tmp);
// update current exporter
currentExporter = &exporterCache.front();
// pop last element if exporter cache is to long
if(exporterCache.size() > MAX_EXPORTER)
exporterCache.pop_back();
return id;
}
/**
* Get data of the record is given by the IPFIX_TYPEID
*/
uint64_t IpfixDbWriterOracle::getData(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
switch (type.length) {
case 1:
return (*(uint8_t*)data);
case 2:
return ntohs(*(uint16_t*)data);
case 4:
return ntohl(*(uint32_t*)data);
case 5: // may occur in the case if IP address + mask
return ntohl(*(uint32_t*)data);
case 8:
return ntohll(*(uint64_t*)data);
default:
printf("Uint with length %d unparseable", type.length);
return 0;
}
}
/***** Public Methods ****************************************************/
/**
* called on Data Record arrival
*/
void IpfixDbWriterOracle::onDataRecord(IpfixDataRecord* record)
{
// only treat non-Options Data Records (although we cannot be sure that there is a Flow inside)
if((record->templateInfo->setId != TemplateInfo::NetflowTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixDataTemplate)) {
record->removeReference();
return;
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Data record received will be passed for processing");
processDataDataRecord(*record->sourceID.get(), *record->templateInfo.get(),
record->dataLength, record->data);
record->removeReference();
}
/**
* Constructor
*/
IpfixDbWriterOracle::IpfixDbWriterOracle(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint32_t observationDomainId, unsigned maxStatements,
const vector<string>& columns)
: currentExporter(NULL), numberOfInserts(0), maxInserts(maxStatements),
dbHost(hostname), dbName(dbname), dbUser(username), dbPassword(password), dbPort(port), con(0)
{
int i;
// set default source id
srcId.exporterAddress.len = 0;
srcId.observationDomainId = observationDomainId;
srcId.exporterPort = 0;
srcId.receiverPort = 0;
srcId.protocol = 0;
srcId.fileDescriptor = 0;
// invalide start settings for current table (to enforce table create)
currentTable.startTime = 1;
currentTable.endTime = 0;
if(columns.empty())
THROWEXCEPTION("IpfixDbWriterOracle: cannot initiate with no columns");
/* get columns */
bool first = true;
for(vector<string>::const_iterator col = columns.begin(); col != columns.end(); col++) {
i = 0;
while(identify[i].columnName != 0) {
if(col->compare(identify[i].columnName) == 0) {
Column c = identify[i];
tableColumns.push_back(c);
// update tableColumnsString
if(!first)
tableColumnsString.append(",");
tableColumnsString.append(identify[i].columnName);
// update tableColumnsCreateString
if(!first)
tableColumnsCreateString.append(", ");
tableColumnsCreateString.append(identify[i].columnName);
tableColumnsCreateString.append(" ");
tableColumnsCreateString.append(identify[i].columnType);
first = false;
break;
}
i++;
}
}
msg(MSG_INFO, "IpfixDbWriterOracle: columns are %s", tableColumnsString.c_str());
if(connectToDB() != 0)
THROWEXCEPTION("IpfixDbWriterOracle creation failed");
}
/**
* Destructor
*/
IpfixDbWriterOracle::~IpfixDbWriterOracle()
{
writeToDb();
env->terminateConnection(con);
oracle::occi::Environment::terminateEnvironment(env);
}
#endif /* ORACLE_SUPPORT_ENABLED */

137
src/modules/ipfix/IpfixDbWriterOracle.hpp
View File

@ -1,137 +0,0 @@
/*
* IPFIX Database Reader/Writer Oracle Connector
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef ORACLE_SUPPORT_ENABLED
#ifndef IPFIXDBWRITERORACLE_H_
#define IPFIXDBWRITERORACLE_H_
#include "IpfixDbCommon.hpp"
#include "IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <occi.h>
#include <iostream>
#include <iomanip>
#include <stdlib.h>
#include <stdio.h>
#include <netinet/in.h>
#include <time.h>
#include <sstream>
using namespace std;
#define EXPORTERID 0
/**
* IpfixDbWriterOracle powered the communication to the oracle database server
* also between the other structs
*/
class IpfixDbWriterOracle
: public IpfixRecordDestination, public Module, public Source<NullEmitable*>
{
public:
IpfixDbWriterOracle(const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint32_t observationDomainId, unsigned maxStatements,
const vector<string>& columns);
~IpfixDbWriterOracle();
void onDataRecord(IpfixDataRecord* record);
/**
* Struct to identify the relationship between columns names and
* IPFIX_TYPEID, column type and default value
*/
struct Column {
const char* columnName; /** column name */
const char* columnType; /** column data type in database */
uint64_t defaultValue; /** default value */
InformationElement::IeId ipfixId; /** IPFIX_TYPEID */
InformationElement::IeEnterpriseNumber enterprise; /** enterprise number */
};
private:
static const unsigned MAX_EXPORTER = 10; // maximum numbers of cached exporters
/**
* Struct buffers start and end time and tablename for the different tables
*/
struct TableCacheEntry {
time_t startTime; // smallest flow start second timestamp in the table
time_t endTime; // largest flow start second timestamp in the table
string name; // name of the table
};
/**
* Struct buffers ODID, IP address and row index of an exporter
*/
struct ExporterCacheEntry {
IpfixRecord::SourceID sourceID;/** source id of the exporter */
int id; /** Id entry of sourcID and expIP in the ExporterTable */
};
TableCacheEntry currentTable; // current table in tableCache
list<ExporterCacheEntry> exporterCache; // cached tables names, key=observationDomainId
ExporterCacheEntry* currentExporter; // pointer to current exporter in exporterCache
IpfixRecord::SourceID srcId; // default source ID
ostringstream insertStatement; // insert statement string
int numberOfInserts; // number of inserts in statement
int maxInserts; // maximum number of inserts per statement
vector<Column> tableColumns; // table columns
string tableColumnsString; // table columns as string for INSERT statements
string tableColumnsCreateString; // table columns as string for CREATE statements
// database data
string dbHost, dbName, dbUser, dbPassword;
unsigned dbPort;
oracle::occi::Environment *env;
oracle::occi::Connection *con;
bool dbError; // db error flag
int createDB();
int setCurrentTable(time_t flowstartsec);
string& getInsertString(string& row, time_t& flowstartsec, const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data);
int writeToDb();
int getExporterID(const IpfixRecord::SourceID& sourceID);
int connectToDB();
int createExporterTable();
void processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data);
uint64_t getData(InformationElement::IeInfo type, IpfixRecord::Data* data);
bool equalExporter(const IpfixRecord::SourceID& a, const IpfixRecord::SourceID& b);
const static Column identify[];
};
#endif
#endif

108
src/modules/ipfix/IpfixDbWriterOracleCfg.cpp
View File

@ -1,108 +0,0 @@
/*
* IPFIX Database Reader/Writer Oracle Connector Configuration
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include "IpfixDbWriterOracleCfg.h"
IpfixDbWriterOracleCfg* IpfixDbWriterOracleCfg::create(XMLElement* e)
{
assert(e);
assert(e->getName() == getName());
return new IpfixDbWriterOracleCfg(e);
}
IpfixDbWriterOracleCfg::IpfixDbWriterOracleCfg(XMLElement* elem)
: CfgHelper<IpfixDbWriterOracle, IpfixDbWriterOracleCfg>(elem, "ipfixDbWriterOracle"),
port(0), bufferRecords(30), observationDomainId(0)
{
msg(MSG_DEBUG, "Starting configuration for Oracle connection");
if (!elem) return;
XMLNode::XMLSet<XMLElement*> set = _elem->getElementChildren();
for (XMLNode::XMLSet<XMLElement*>::iterator it = set.begin(); it != set.end(); it++) {
XMLElement* e = *it;
if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
} else if (e->matches("dbname")) {
dbname = e->getFirstText();
} else if (e->matches("username")) {
user = e->getFirstText();
} else if (e->matches("password")) {
password = e->getFirstText();
} else if (e->matches("bufferrecords")) {
bufferRecords = getInt("bufferrecords");
} else if (e->matches("columns")) {
readColumns(e);
} else if (e->matches("next")) { // ignore next
} else {
msg(MSG_FATAL, "Unknown IpfixDbWriterOracle config statement %s\n", e->getName().c_str());
continue;
}
}
if (hostname=="") THROWEXCEPTION("IpfixDbWriterOracleCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbWriterOracleCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbWriterOracleCfg: dbname not set in configuration!");
if (user=="") THROWEXCEPTION("IpfixDbWriterOracleCfg: username not set in configuration!");
if (password=="") THROWEXCEPTION("IpfixDbWriterOracleCfg: password not set in configuration!");
}
void IpfixDbWriterOracleCfg::readColumns(XMLElement* elem) {
colNames.clear();
XMLNode::XMLSet<XMLElement*> set = elem->getElementChildren();
for (XMLNode::XMLSet<XMLElement*>::iterator it = set.begin();
it != set.end();
it++) {
XMLElement* e = *it;
if (e->matches("name")) {
colNames.push_back(e->getFirstText());
msg(MSG_DEBUG, "Row: %s", e->getFirstText().c_str());
} else {
msg(MSG_FATAL, "Unknown IpfixDbWriterOracle config statement %s\n", e->getName().c_str());
continue;
}
}
}
IpfixDbWriterOracleCfg::~IpfixDbWriterOracleCfg()
{
}
IpfixDbWriterOracle* IpfixDbWriterOracleCfg::createInstance()
{
instance = new IpfixDbWriterOracle(hostname, dbname, user, password, port, observationDomainId, bufferRecords, colNames);
msg(MSG_DEBUG, "IpfixDbWriterOracle configuration host %s db %s user %s password %s port %i observationDomainId %i bufferRecords %i\n",
hostname.c_str(), dbname.c_str(), user.c_str(), password.c_str(), port, observationDomainId, bufferRecords);
return instance;
}
bool IpfixDbWriterOracleCfg::deriveFrom(IpfixDbWriterOracleCfg* old)
{
return false;
}
#endif /* ORACLE_SUPPORT_ENABLED */

64
src/modules/ipfix/IpfixDbWriterOracleCfg.h
View File

@ -1,64 +0,0 @@
/*
* IPFIX Database Reader/Writer Oracle Connector Configuration
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifndef IPFIXDBWRITERORACLECFG_H_
#define IPFIXDBWRITERORACLECFG_H_
#ifdef ORACLE_SUPPORT_ENABLED
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "IpfixDbWriterOracle.hpp"
#include <string>
using namespace std;
class IpfixDbWriterOracleCfg
: public CfgHelper<IpfixDbWriterOracle, IpfixDbWriterOracleCfg>
{
public:
friend class ConfigManager;
virtual IpfixDbWriterOracleCfg* create(XMLElement* e);
virtual ~IpfixDbWriterOracleCfg();
virtual IpfixDbWriterOracle* createInstance();
virtual bool deriveFrom(IpfixDbWriterOracleCfg* old);
protected:
string hostname; /**< hostname of database host */
uint16_t port; /**< port of database */
string dbname; /**< database name */
string user; /**< user name for login to database */
string password; /**< password for login to database */
uint16_t bufferRecords; /**< amount of records to buffer until they are written to database */
uint32_t observationDomainId; /**< default observation domain id (overrides the one received in the records */
vector<string> colNames; /**< column names */
void readColumns(XMLElement*);
IpfixDbWriterOracleCfg(XMLElement*);
};
#endif /* ORACLE_SUPPORT_ENABLED */
#endif /* IPFIXDBWRITERORACLECFG_H_ */

861
src/modules/ipfix/IpfixDbWriterPg.cpp
View File

@ -1,861 +0,0 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007 Gerhard Muenz
* Copyright (C) 2008 Tobias Limmer
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriterPg and IpfixDbReader */
#ifdef PG_SUPPORT_ENABLED
#include "IpfixDbWriterPg.hpp"
#include "common/msg.h"
#include "common/Misc.h"
#include "common/Time.h"
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include <sstream>
#include <algorithm>
using namespace std;
/***** Global Variables ******************************************************/
/**
* maximum length of one item in a SQL statement
*/
const uint16_t MAX_COL_LENGTH = 22;
/**
* struct to identify column names with IPFIX_TYPEID an the dataType to store in database
* ExporterID is no IPFIX_TYPEID, its user specified
* Attention: order of entries is important!
*/
const IpfixDbWriterPg::Column IpfixDbWriterPg::identify [] = {
{ "srcIP", IPFIX_TYPEID_sourceIPv4Address, "inet", 0, 0},
{ "dstIP", IPFIX_TYPEID_destinationIPv4Address, "inet", 0, 0},
{ "srcPort", IPFIX_TYPEID_sourceTransportPort, "integer", 0, 0},
{ "dstPort", IPFIX_TYPEID_destinationTransportPort, "integer",0, 0},
{ "proto",IPFIX_TYPEID_protocolIdentifier , "smallint", 0, 0},
{ "dstTos", IPFIX_TYPEID_classOfServiceIPv4, "smallint", 0, 0},
{ "bytes", IPFIX_TYPEID_octetDeltaCount, "bigint", 0, 0},
{ "pkts", IPFIX_TYPEID_packetDeltaCount, "bigint", 0, 0},
{ "firstSwitched", IPFIX_TYPEID_flowStartMilliSeconds, "timestamp", 0, 0}, // default value is invalid/not used for this entry
{ "lastSwitched", IPFIX_TYPEID_flowEndMilliSeconds, "timestamp", 0, 0}, // default value is invalid/not used for this entry
{ "tcpControlBits", IPFIX_TYPEID_tcpControlBits, "smallint", 0, 0},
{ "revbytes", IPFIX_TYPEID_octetDeltaCount, "bigint", IPFIX_PEN_reverse, 0},
{ "revpkts", IPFIX_TYPEID_packetDeltaCount, "bigint", IPFIX_PEN_reverse, 0},
{ "revFirstSwitched", IPFIX_TYPEID_flowStartMilliSeconds, "timestamp", IPFIX_PEN_reverse, 0}, // default value is invalid/not used for this entry
{ "revLastSwitched", IPFIX_TYPEID_flowEndMilliSeconds, "timestamp", IPFIX_PEN_reverse, 0}, // default value is invalid/not used for this entry
{ "revTcpControlBits", IPFIX_TYPEID_tcpControlBits, "smallint", IPFIX_PEN_reverse, 0},
{ "maxPacketGap", IPFIX_ETYPEID_maxPacketGap, "bigint", IPFIX_PEN_vermont|IPFIX_PEN_reverse, 0},
{ "revMaxPacketGap", IPFIX_ETYPEID_maxPacketGap, "bigint", IPFIX_PEN_vermont|IPFIX_PEN_reverse, 0},
{ "exporterID",EXPORTERID, "integer", 0},
{ 0} // last entry must be 0
};
/**
* (re)connect to database
*/
void IpfixDbWriterPg::connectToDB()
{
dbError = true;
// close (in the case that it was already connected)
if (conn) PQfinish(conn);
/**Connect to Database*/
ostringstream conninfo;
conninfo << "host='" << hostName << "' port='" << portNum << "' ";
conninfo << "dbname='" << dbName << "' user='" << userName<< "' ";
conninfo << "password='" << password << "' sslmode=require";
DPRINTF("using connection string '%s'", conninfo.str().c_str());
conn = PQconnectdb(conninfo.str().c_str());
/*if (!mysql_real_connect(conn,
hostName, userName,
password, 0, portNum,
socketName, flags)) {
msg(MSG_FATAL,"IpfixDbWriterPg: Connection to database failed. Error: %s",
mysql_error(conn));
return;
} else {
msg(MSG_DEBUG,"IpfixDbWriterPg succesfully connected to database");
}*/
if (PQstatus(conn) != CONNECTION_OK) {
msg(MSG_FATAL, "IpfixDbWriterPg: Connection to database failed, error: %s", PQerrorMessage(conn));
return;
}
/**create table exporter*/
if (createExporterTable()!=0) return;
dbError = false;
}
int IpfixDbWriterPg::createExporterTable()
{
/**is there already a table with the same name - drop it */
/* gerhard: let's keep the database, we do not want to lose data
char dropExporterTab[STARTLEN];
strcpy(dropExporterTab,"DROP TABLE IF EXISTS exporter");
if(mysql_query(conn, dropExporterTab) != 0) {
msg(MSG_FATAL,"Drop of exists exporter table failed. Error: %s",
mysql_error(conn));
return 1;
}
*/
/** create table exporter*/
ostringstream oss;
oss << "SELECT COUNT(*) FROM pg_class where relname='exporter'";
PGresult* res = PQexec(conn, oss.str().c_str());
DPRINTF("PQntuples: %d", PQntuples(res));
if((PQresultStatus(res) != PGRES_TUPLES_OK) || (PQntuples(res)==0)) {
msg(MSG_FATAL, "IpfixDbWriterPg: Failed to check if table 'exporter' exists. Error: %s",
PQerrorMessage(conn));
PQclear(res);
return 1;
}
if (atoi(PQgetvalue(res, 0, 0))!=1) {
string ctexporter = "CREATE TABLE exporter (id serial PRIMARY KEY, "
"sourceID integer, "
"srcIP inet)";
res = PQexec(conn, ctexporter.c_str());
if(PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL, "IpfixDbWriterPg: Creation of table Exporter failed. Error: %s",
PQerrorMessage(conn));
PQclear(res);
return 1;
} else {
PQclear(res);
msg(MSG_DEBUG, "Exporter table created");
}
}
return 0;
}
bool IpfixDbWriterPg::checkRelationExists(const char* relname)
{
// check if table needs to be created
ostringstream oss;
oss << "SELECT COUNT(*) FROM pg_class where relname='" << relname << "'";
PGresult* res = PQexec(conn, oss.str().c_str());
if((PQresultStatus(res) != PGRES_TUPLES_OK) || (PQntuples(res)==0)) {
msg(MSG_FATAL, "IpfixDbWriterPg: Failed to check if relation '%s' exists. Error: %s",
relname, PQerrorMessage(conn));
PQclear(res);
return false;
}
if (atoi(PQgetvalue(res, 0, 0))==1) {
PQclear(res);
return true;
}
PQclear(res);
return false;
}
/**
* Create a table in the database
*/
bool IpfixDbWriterPg::createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime)
{
uint32_t i;
if (find(usedPartitions.begin(), usedPartitions.end(), partitionname)!=usedPartitions.end()) {
// found cached entry!
DPRINTF("Partition '%s' already created.", partitionname);
return true;
}
ostringstream ctsql;
ostringstream oss;
// check if table needs to be created
if (!checkRelationExists(tablePrefix.c_str())) {
ctsql << "CREATE TABLE " << tablePrefix << " (";
/**collect the names for columns and the dataTypes for the table in a string*/
for(i=0; i < numberOfColumns; i++) {
ctsql << identify[i].cname << " " << identify[i].dataType;
if (i != numberOfColumns-1) {
ctsql << ", ";
}
}
ctsql << ")";
/** create table*/
PGresult* res = PQexec(conn, ctsql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of table failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Table %s created ", tablePrefix.c_str());
}
}
if (!checkRelationExists(partitionname)) {
// create partition
ostringstream cpsql;
cpsql << "CREATE TABLE " << partitionname << " (CHECK (firstswitched>='";
cpsql << getTimeAsString(starttime, "%Y-%m-%d %H:%M:%S", true);
cpsql << "' AND firstswitched<'" << getTimeAsString(endtime, "%Y-%m-%d %H:%M:%S", true);
cpsql << "')) INHERITS (" << tablePrefix << ")";
PGresult* res = PQexec(conn, cpsql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of partition failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Partition %s created ", partitionname);
usedPartitions.push_back(partitionname);
if (usedPartitions.size()>MAX_USEDTABLES) usedPartitions.pop_front();
}
}
string indexname = string(partitionname) + "_firstswitched";
if (!checkRelationExists(indexname.c_str())) {
ostringstream cisql;
cisql << "CREATE INDEX " << indexname <<" ON " << partitionname;
cisql << "(firstswitched)";
PGresult* res = PQexec(conn, cisql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of index failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Index %s_firstswitched created ", partitionname);
}
}
return true;
}
/**
* save given elements of record to database
*/
void IpfixDbWriterPg::processDataDataRecord(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data)
{
DPRINTF("Processing data record");
if (dbError) {
connectToDB();
if (dbError) return;
}
/** check if statement buffer is not full*/
if(insertBuffer.curRows==insertBuffer.maxRows) {
msg(MSG_ERROR, "failed to write data to database, trying again ...");
if (!writeToDb()) {
msg(MSG_ERROR, "dropping record");
return;
}
}
/** sourceid null ? use default*/
/* overwrite sourceid if defined */
if(srcId.observationDomainId != 0 || sourceID == NULL) {
sourceID = &srcId;
}
// insert record into buffer
fillInsertRow(sourceID, dataTemplateInfo, length, data);
// statemBuffer is filled -> insert in table
if(insertBuffer.curRows==insertBuffer.maxRows) {
msg(MSG_INFO, "Writing buffered records to database");
writeToDb();
}
}
/**
* function receive the when callback is started
*/
void IpfixDbWriterPg::onDataRecord(IpfixDataRecord* record)
{
// only treat non-Options Data Records (although we cannot be sure that there is a Flow inside)
if((record->templateInfo->setId != TemplateInfo::NetflowTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixDataTemplate)) {
record->removeReference();
return;
}
processDataDataRecord(record->sourceID.get(), record->templateInfo.get(),
record->dataLength, record->data);
record->removeReference();
}
bool IpfixDbWriterPg::checkCurrentTable(uint64_t flowStart)
{
return curTable.timeStart!=0 && (curTable.timeStart<=flowStart && curTable.timeEnd>flowStart);
}
string IpfixDbWriterPg::getTimeAsString(uint64_t milliseconds, const char* formatstring, bool addfraction, uint32_t microseconds)
{
char timebuffer[40];
struct tm date;
time_t seconds = milliseconds/1000; // round down to start of interval and convert ms -> s
gmtime_r(&seconds, &date);
strftime(timebuffer, ARRAY_SIZE(timebuffer), formatstring, &date);
if (addfraction) {
#if DEBUG
if (ARRAY_SIZE(timebuffer)-strlen(timebuffer)<7) THROWEXCEPTION("IpfixDbWriterPg: buffer size too small");
#endif
snprintf(timebuffer+strlen(timebuffer), 8, ".%06u", static_cast<uint32_t>(milliseconds%1000)*1000+(microseconds%1000));
}
return string(timebuffer);
}
bool IpfixDbWriterPg::setCurrentTable(uint64_t flowStart)
{
if (insertBuffer.curRows) THROWEXCEPTION("programming error: setCurrentTable MUST NOT be called when entries are still cached!");
string tablename = tablePrefix;
uint64_t starttime = (flowStart/TABLE_INTERVAL)*TABLE_INTERVAL; // round down to start of interval
uint64_t endtime = starttime+TABLE_INTERVAL;
tablename += getTimeAsString(starttime, "_%y%m%d_%H%M%S", false);
if (!createDBTable(tablename.c_str(), starttime, endtime)) return false;
// build SQL INSERT statement
ostringstream sql;
sql << "INSERT INTO " << tablename << " (";
for (uint32_t i=0; i<numberOfColumns; i++) {
sql << identify[i].cname;
if (i<numberOfColumns-1) sql << ", ";
}
sql << ") VALUES ";
strcpy(insertBuffer.sql, sql.str().c_str());
insertBuffer.bodyPtr = insertBuffer.sql + sql.str().size();
insertBuffer.appendPtr = insertBuffer.bodyPtr;
curTable.name = tablename;
curTable.timeStart = starttime;
curTable.timeEnd = endtime;
return true;
}
// extract seconds, ms and ys from ntp time
void IpfixDbWriterPg::extractNtp64(uint64_t& intdata, uint32_t& micros)
{
if (intdata==0) {
micros = 0;
return;
}
timeval t = timentp64(*((ntp64*)(&intdata)));
intdata = (uint64_t)t.tv_sec*1000+t.tv_usec/1000;
micros = t.tv_usec%1000;
}
/**
* loop over the TemplateInfo (fieldinfo,datainfo) to get the IPFIX values to store in database
* results are stored in insertBuffer.sql
*/
void IpfixDbWriterPg::fillInsertRow(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length, IpfixRecord::Data* data)
{
uint32_t j, k;
uint64_t intdata = 0;
ostringstream insertsql;
uint64_t flowstart = 0;
insertsql << "(";
/**loop over the columname and loop over the IPFIX_TYPEID of the record
to get the corresponding data to store */
for( j=0; j<numberOfColumns; j++) {
bool notfound = true;
uint32_t microseconds = 0; // special case for handle of microseconds (will not be contained in intdata)
if (identify[j].ipfixId == EXPORTERID) {
/**lookup exporter buffer to get exporterID from sourcID and expIp**/
uint32_t expID = getExporterID(sourceID);
intdata = expID;
notfound = false;
} else {
// try to gather data required for the field
if(dataTemplateInfo->fieldCount > 0) {
// look inside the ipfix data packet
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type.id == identify[j].ipfixId) {
notfound = false;
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
DPRINTF("IpfixDbWriterPg::getRecData: really saw ipfix id %d in packet with intdata %llX, type %d, length %d and offset %X", identify[j].ipfixId, intdata, dataTemplateInfo->fieldInfo[k].type.id, dataTemplateInfo->fieldInfo[k].type.length, dataTemplateInfo->fieldInfo[k].offset);
}
}
}
if( dataTemplateInfo->dataCount > 0 && notfound) {
// look in static data fields of template for data
for(k=0; k < dataTemplateInfo->dataCount; k++) {
if(dataTemplateInfo->dataInfo[k].type.id == identify[j].ipfixId) {
notfound = false;
intdata = getdata(dataTemplateInfo->dataInfo[k].type,(dataTemplateInfo->data+dataTemplateInfo->dataInfo[k].offset));
}
}
}
if(notfound) {
// for some Ids, we have an alternative
if (identify[j].enterprise==0) {
switch (identify[j].ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
// look for alternative (flowStartMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartSeconds, 0)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
} else if (dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartNanoSeconds, 0)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
extractNtp64(intdata, microseconds);
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
// look for alternative (flowEndMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndSeconds, 0)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
} else if (dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndNanoSeconds, 0)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
extractNtp64(intdata, microseconds);
notfound = false;
break;
}
}
}
break;
}
} else if (identify[j].enterprise==IPFIX_PEN_reverse) {
switch (identify[j].ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
// look for alternative (revFlowStartMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
} else if (dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartNanoSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
extractNtp64(intdata, microseconds);
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
// look for alternative (revFlowEndMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
} else if (dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndNanoSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
extractNtp64(intdata, microseconds);
notfound = false;
break;
}
}
}
break;
}
}
// if still not found, get default value
if(notfound)
intdata = getdefaultIPFIXdata(identify[j].ipfixId);
}
// we need extra treatment for timing related fields
switch (identify[j].ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
if(intdata==0) {
// if no flow start time is available, maybe this is is from a netflow from Cisco
// then use flowStartSysUpTime as flow start time
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSysUpTime) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset))*1000;
}
}
}
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
if(intdata==0) {
// if no flow end time is available, maybe this is is from a netflow from Cisco
// then use flowEndSysUpTime as flow start time
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSysUpTime) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset))*1000;
}
}
}
break;
}
}
DPRINTF("saw ipfix id %d in packet with intdata %llX", identify[j].ipfixId, intdata);
switch (identify[j].enterprise) {
case 0:
switch (identify[j].ipfixId) {
// convert IPv4 address to string notation, as this is required by Postgres
case IPFIX_TYPEID_sourceIPv4Address:
case IPFIX_TYPEID_destinationIPv4Address:
insertsql << "'" << IPToString(ntohl(intdata)) << "'";
break;
// convert integer to timestamps
case IPFIX_TYPEID_flowStartMilliSeconds:
// save time for table access
if (flowstart==0) flowstart = intdata;
case IPFIX_TYPEID_flowEndMilliSeconds:
insertsql << "'" << getTimeAsString(intdata, "%Y-%m-%d %H:%M:%S", true, microseconds) << "'";
break;
// all other integer data is directly converted to a string
default:
insertsql << intdata;
break;
}
break;
case IPFIX_PEN_reverse:
switch (identify[j].ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
insertsql << "'" << getTimeAsString(intdata, "%Y-%m-%d %H:%M:%S", true, microseconds) << "'";
break;
// all other integer data is directly converted to a string
default:
insertsql << intdata;
break;
}
break;
// all other integer data is directly converted to a string
default:
insertsql << intdata;
break;
}
if (j!=numberOfColumns-1) insertsql << ",";
}
insertsql << "),";
// if this flow belongs to a different table, flush all cached entries now
// and get new table
if (!checkCurrentTable(flowstart)) {
if (!writeToDb()) {
msg(MSG_ERROR, "failed to flush table, dropping record");
return;
}
if (!setCurrentTable(flowstart)) {
msg(MSG_ERROR, "failed to change table, dropping record");
return;
}
}
strcat(insertBuffer.appendPtr, insertsql.str().c_str());
insertBuffer.appendPtr += insertsql.str().size();
insertBuffer.curRows++;
}
/**
* Function writes the content of the statemBuffer to database
* statemBuffer consist of single insert statements
*/
bool IpfixDbWriterPg::writeToDb()
{
if (insertBuffer.curRows==0) return true;
// delete last comma from sql string, as it is always inserted by fillRowColumns
insertBuffer.appendPtr[-1] = 0;
// Write rows to database
PGresult* res = PQexec(conn, insertBuffer.sql);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Insert of records failed. Error: %s",
PQerrorMessage(conn));
PQclear(res);
goto dbwriteerror;
}
PQclear(res);
insertBuffer.curRows = 0;
insertBuffer.appendPtr = insertBuffer.bodyPtr;
*insertBuffer.appendPtr = 0;
msg(MSG_DEBUG,"Write to database is complete");
return true;
dbwriteerror:
dbError = true;
return false;
}
/**
* Returns the exporterID
* For every different sourcID and expIp a unique ExporterID will be generated from the database
* First lookup for the ExporterID in the exporterBuffer according sourceID and expIp, is there nothing
* lookup in the ExporterTable, is there also nothing insert sourceID and expIp an return the generated
* ExporterID
*/
int IpfixDbWriterPg::getExporterID(IpfixRecord::SourceID* sourceID)
{
uint32_t i;
int exporterID = 0;
char statementStr[EXPORTER_WIDTH];
uint32_t expIp = 0;
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID->exporterAddress.toUInt32();
/** Is the exporterID already in exporterBuffer? */
for(i = 0; i < curExporterEntries; i++) {
if(exporterEntries[i].observationDomainId == sourceID->observationDomainId &&
exporterEntries[i].ip==expIp) {
DPRINTF("Exporter sourceID/IP with ID %d is in the exporterBuffer\n",
exporterEntries[i].Id);
return exporterEntries[i].Id;
}
}
// it is not: try to get it from the database
sprintf(statementStr, "SELECT id FROM exporter WHERE sourceID=%u AND srcIp='%s'", sourceID->observationDomainId, IPToString(expIp).c_str());
PGresult* res = PQexec(conn, statementStr);
if (PQresultStatus(res) != PGRES_TUPLES_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Select on exporter table failed. Error: %s",
PQerrorMessage(conn));
return 0;// If a failure occurs, return exporterID = 0
}
/** is the exporterID in the exporter table ?*/
if (PQntuples(res)) {
exporterID = atoi(PQgetvalue(res, 0, 0));
DPRINTF("ExporterID %d is in exporter table\n",exporterID);
}
else
{
PQclear(res);
/**ExporterID is not in exporter table - insert expID and expIp and return the exporterID*/
sprintf(statementStr, "INSERT INTO exporter (sourceID, srcIP) VALUES ('%u','%s') RETURNING id",
sourceID->observationDomainId, IPToString(expIp).c_str());
res = PQexec(conn, statementStr);
if(PQresultStatus(res) != PGRES_TUPLES_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Insert in exporter table failed. Error: %s",
PQerrorMessage(conn));
return 0;
}
exporterID = atoi(PQgetvalue(res, 0, 0));
msg(MSG_INFO,"ExporterID %d inserted in exporter table", exporterID);
}
PQclear(res);
if (curExporterEntries==MAX_EXP_TABLE-1) {
// maybe here we should check how often this happens and display a severe warning if too
// many parallel streams are received at once
msg(MSG_INFO, "IpfixDbWriterPg: turnover for exporter cache occurred.");
curExporterEntries = 0;
}
/**Write new exporter in the exporterBuffer*/
exporterEntries[curExporterEntries].Id = exporterID;
exporterEntries[curExporterEntries].observationDomainId = sourceID->observationDomainId;
exporterEntries[curExporterEntries++].ip = expIp;
return exporterID;
}
/**
* Get data of the record is given by the IPFIX_TYPEID
*/
uint64_t IpfixDbWriterPg::getdata(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
if(type.id == IPFIX_TYPEID_sourceIPv4Address || type.id == IPFIX_TYPEID_destinationIPv4Address)
return getipv4address(type, data);
else
return getIPFIXValue(type, data);
}
/**
* determine the ipv4address of the data record
*/
uint32_t IpfixDbWriterPg::getipv4address( InformationElement::IeInfo type, IpfixRecord::Data* data)
{
if (type.length > 5) {
DPRINTF("IPv4 Address with length %d unparseable\n", type.length);
return 0;
}
if ((type.length == 5) && ( type.id == IPFIX_TYPEID_sourceIPv4Address || IPFIX_TYPEID_destinationIPv4Address )) /*&& (imask != 0)*/{
DPRINTF("imask drop from ipaddress\n");
type.length = 4;
}
if ((type.length < 5) &&( type.id == IPFIX_TYPEID_sourceIPv4Address || type.id == IPFIX_TYPEID_destinationIPv4Address)) /*&& (imask == 0)*/{
return getIPFIXValue(type, data);
}
return 0;
}
/**
* get the IPFIX value
*/
uint64_t IpfixDbWriterPg::getIPFIXValue(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
switch (type.length) {
case 1:
return (*(uint8_t*)data);
case 2:
return ntohs(*(uint16_t*)data);
case 4:
return ntohl(*(uint32_t*)data);
case 8:
return ntohll(*(uint64_t*)data);
default:
printf("Uint with length %d unparseable\n", type.length);
return 0;
}
}
/**
* if there no IPFIX_TYPEID in the given data record
* get the default value to store in the database columns
*/
uint32_t IpfixDbWriterPg::getdefaultIPFIXdata(int ipfixtype_id)
{
int i;
for( i=0; identify[i].cname != 0; i++) {
if(ipfixtype_id == identify[i].ipfixId) {
return identify[i].defaultValue;
}
}
return 0;
}
/***** Exported Functions ****************************************************/
/**
* Creates a new IpfixDbWriterPg. Do not forget to call @c startipfixDbWriter() to begin writing to Database
* @return handle to use when calling @c destroyipfixDbWriter()
*/
IpfixDbWriterPg::IpfixDbWriterPg(const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId,
int maxStatements)
{
/**Initialize structure members IpfixDbWriterPg*/
hostName = host;
dbName = db;
userName = user;
password = pw;
portNum = port;
conn = 0;
socketName = 0;
flags = 0;
srcId.exporterAddress.len = 0;
srcId.observationDomainId = observationDomainId;
srcId.exporterPort = 0;
srcId.receiverPort = 0;
srcId.protocol = 0;
srcId.fileDescriptor = 0;
bzero(&exporterEntries, sizeof(exporterEntries));
curExporterEntries = 0;
curTable.timeStart = 0;
curTable.timeEnd = 0;
curTable.name = "";
tablePrefix = "f"; // TODO: make this in config file configurable!
/**count columns*/
numberOfColumns = 0;
for(uint32_t i=0; identify[i].cname!=0; i++) numberOfColumns++;
/**Initialize structure members Statement*/
insertBuffer.curRows = 0;
insertBuffer.maxRows = maxStatements;
insertBuffer.sql = new char[(INS_WIDTH+3)*(numberOfColumns+1)*maxStatements+numberOfColumns*20+60+1];
*insertBuffer.sql = 0;
connectToDB();
}
/**
* Frees memory used by an IpfixDbWriterPg
* @param IpfixDbWriterPg handle obtained by calling @c createipfixDbWriter()
*/
IpfixDbWriterPg::~IpfixDbWriterPg()
{
writeToDb();
if (conn) PQfinish(conn);
delete[] insertBuffer.sql;
}
#endif

90
src/modules/ipfix/IpfixDbWriterPgCfg.cpp
View File

@ -1,90 +0,0 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef PG_SUPPORT_ENABLED
#include "IpfixDbWriterPgCfg.h"
IpfixDbWriterPgCfg* IpfixDbWriterPgCfg::create(XMLElement* e)
{
assert(e);
assert(e->getName() == getName());
return new IpfixDbWriterPgCfg(e);
}
IpfixDbWriterPgCfg::IpfixDbWriterPgCfg(XMLElement* elem)
: CfgHelper<IpfixDbWriterPg, IpfixDbWriterPgCfg>(elem, "ipfixDbWriterPg"),
port(0),
bufferRecords(30)
{
if (!elem) return;
XMLNode::XMLSet<XMLElement*> set = _elem->getElementChildren();
for (XMLNode::XMLSet<XMLElement*>::iterator it = set.begin();
it != set.end();
it++) {
XMLElement* e = *it;
if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
} else if (e->matches("dbname")) {
dbname = e->getFirstText();
} else if (e->matches("username")) {
user = e->getFirstText();
} else if (e->matches("password")) {
password = e->getFirstText();
} else if (e->matches("bufferrecords")) {
bufferRecords = getInt("bufferrecords");
} else if (e->matches("next")) { // ignore next
} else {
msg(MSG_FATAL, "Unknown IpfixDbWriterPg config statement %s\n", e->getName().c_str());
continue;
}
}
if (hostname=="") THROWEXCEPTION("IpfixDbWriterPgCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbWriterPgCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbWriterPgCfg: dbname not set in configuration!");
if (user=="") THROWEXCEPTION("IpfixDbWriterPgCfg: username not set in configuration!");
if (password=="") THROWEXCEPTION("IpfixDbWriterPgCfg: password not set in configuration!");
}
IpfixDbWriterPgCfg::~IpfixDbWriterPgCfg()
{
}
IpfixDbWriterPg* IpfixDbWriterPgCfg::createInstance()
{
instance = new IpfixDbWriterPg(hostname.c_str(), dbname.c_str(), user.c_str(), password.c_str(), port, 0, bufferRecords);
return instance;
}
bool IpfixDbWriterPgCfg::deriveFrom(IpfixDbWriterPgCfg* old)
{
return false;
}
#endif /*DB_SUPPORT_ENABLED*/

12
src/modules/ipfix/IpfixFileWriter.cpp
View File

@ -28,6 +28,9 @@
#include <stdexcept>
#include <string.h>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
/* go back to SENDER_TEMPLATE_ID_LOW if _HI is reached */
#define SENDER_TEMPLATE_ID_HI 60000
@ -41,11 +44,18 @@ IpfixFileWriter::IpfixFileWriter(uint16_t observationDomainId, std::string filen
std::string destinationPath, uint32_t maximumFilesize)
: IpfixSender(observationDomainId, MAX_RECORD_RATE)
{
// check if directory base exists
if (!boost::filesystem::is_directory(destinationPath)) {
THROWEXCEPTION("Directory %s does not exists or is not a directory!", destinationPath.c_str());
}
if (filenamePrefix != "") {
if(addCollector(observationDomainId, filenamePrefix, destinationPath, maximumFilesize) != 0) {
THROWEXCEPTION("IpfixFileWriter's Collector addition failed");
return;
}
} else {
THROWEXCEPTION("IpfixFileWriter: no filename prefix given. Prefix is required though!");
}
msg(MSG_DEBUG, "IpfixFileWriter: running");
@ -66,7 +76,7 @@ int IpfixFileWriter::addCollector(uint16_t observationDomainId, std::string file
if(destinationPath.at(destinationPath.length()-1) != '/')
destinationPath += "/";
std::string my_filename = destinationPath + filenamePrefix;
if (maximumFilesize < 0) maximumFilesize = DEFAULTFILESIZE;
if (maximumFilesize <= 0) maximumFilesize = DEFAULTFILESIZE;
if(maximumFilesize < 64)
msg(MSG_ERROR,
"maximum filsize < maximum message length - this could lead to serious problems");

2
src/modules/ipfix/IpfixFileWriterCfg.cpp
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2009-2012 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License

4
src/modules/ipfix/IpfixRecord.cpp
View File

@ -84,16 +84,20 @@ namespace InformationElement {
if (enterprise==0 || enterprise==IPFIX_PEN_reverse) {
switch (id) {
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_flowStartSeconds:
case IPFIX_TYPEID_flowEndSeconds:
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_TYPEID_flowStartNanoSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_sourceIPv4Address:
case IPFIX_TYPEID_destinationIPv4Address:
case IPFIX_TYPEID_bgpSourceAsNumber:
case IPFIX_TYPEID_bgpDestinationAsNumber:
case IPFIX_ETYPEID_maxPacketGap:
return Packet::IPProtocolType(Packet::TCP|Packet::UDP|Packet::ICMP);

20
src/modules/ipfix/aggregator/BaseHashtable.cpp
View File

@ -353,9 +353,11 @@ int BaseHashtable::isToBeAggregated(InformationElement::IeInfo& type)
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_TYPEID_flowEndMicroSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_postOctetDeltaCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_postPacketDeltaCount:
case IPFIX_TYPEID_droppedOctetDeltaCount:
case IPFIX_TYPEID_droppedPacketDeltaCount:
@ -373,7 +375,9 @@ int BaseHashtable::isToBeAggregated(InformationElement::IeInfo& type)
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_tcpControlBits:
return 1;
}
@ -502,6 +506,8 @@ void BaseHashtable::genBiflowStructs()
int32_t dstIPIdx = -1;
int32_t srcPortIdx = -1;
int32_t dstPortIdx = -1;
int32_t srcAsIdx = -1;
int32_t dstAsIdx = -1;
uint32_t maxFieldSize = 0;
@ -536,6 +542,14 @@ void BaseHashtable::genBiflowStructs()
dstPortIdx = i;
mapReverseElement(fi->type);
break;
case IPFIX_TYPEID_bgpSourceAsNumber:
srcAsIdx = i;
mapReverseElement(InformationElement::IeInfo(IPFIX_TYPEID_bgpSourceAsNumber, 0));
break;
case IPFIX_TYPEID_bgpDestinationAsNumber:
dstAsIdx = i;
mapReverseElement(InformationElement::IeInfo(IPFIX_TYPEID_bgpDestinationAsNumber, 0));
break;
default:
defaultassign = true;
break;
@ -599,6 +613,12 @@ void BaseHashtable::genBiflowStructs()
case IPFIX_TYPEID_destinationTransportPort:
revKeyMapper[i] = srcPortIdx;
break;
case IPFIX_TYPEID_bgpSourceAsNumber:
revKeyMapper[i] = srcAsIdx;
break;
case IPFIX_TYPEID_bgpDestinationAsNumber:
revKeyMapper[i] = dstAsIdx;
break;
default:
revKeyMapper[i] = i;
break;

4
src/modules/ipfix/aggregator/FlowHashtable.cpp
View File

@ -100,8 +100,10 @@ int FlowHashtable::aggregateField(TemplateInfo::FieldInfo* basefi, TemplateInfo:
*(uint64_t*)baseData = greaterUint64Nbo(*(uint64_t*)baseData, *(uint64_t*)deltaData);
return 0;
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_postOctetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_postPacketDeltaCount:
case IPFIX_TYPEID_droppedOctetDeltaCount:
@ -165,7 +167,9 @@ int FlowHashtable::aggregateField(TemplateInfo::FieldInfo* basefi, TemplateInfo:
*(uint64_t*)baseData = greaterUint64Nbo(*(uint64_t*)baseData, *(uint64_t*)deltaData);
return 0;
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
*(uint64_t*)baseData = addUint64Nbo(*(uint64_t*)baseData, *(uint64_t*)deltaData);
return 0;

31
src/modules/ipfix/aggregator/PacketHashtable.cpp
View File

@ -311,8 +311,10 @@ void (*PacketHashtable::getCopyDataFunction(const ExpFieldData* efd))(CopyFuncPa
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_TYPEID_flowEndMicroSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
if (efd->dstLength != 8) {
THROWEXCEPTION("unsupported length %d for type %s", efd->dstLength, efd->typeId.toString().c_str());
}
@ -325,6 +327,13 @@ void (*PacketHashtable::getCopyDataFunction(const ExpFieldData* efd))(CopyFuncPa
}
break;
case IPFIX_TYPEID_bgpSourceAsNumber:
case IPFIX_TYPEID_bgpDestinationAsNumber:
if (efd->dstLength != 2) {
THROWEXCEPTION("unsupported length %d for type %s", efd->dstLength, efd->typeId.toString().c_str());
}
break;
default:
THROWEXCEPTION("type unhandled by Packet Aggregator: %s", efd->typeId.toString().c_str());
break;
@ -410,6 +419,8 @@ void (*PacketHashtable::getCopyDataFunction(const ExpFieldData* efd))(CopyFuncPa
}
} else if (efd->typeId == IeInfo(IPFIX_TYPEID_packetDeltaCount, 0)) {
return copyDataSetOne;
} else if (efd->typeId == IeInfo(IPFIX_TYPEID_packetTotalCount, 0)) {
return copyDataSetOne;
} else {
return copyDataGreaterLengthNoMod;
}
@ -431,12 +442,14 @@ uint8_t PacketHashtable::getRawPacketFieldLength(const IeInfo& type)
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_tcpControlBits:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_packetTotalCount:
return 1;
case IPFIX_TYPEID_icmpTypeCodeIPv4:
case IPFIX_TYPEID_sourceTransportPort:
case IPFIX_TYPEID_destinationTransportPort:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_octetTotalCount:
return 2;
case IPFIX_TYPEID_flowStartSeconds:
@ -451,6 +464,10 @@ uint8_t PacketHashtable::getRawPacketFieldLength(const IeInfo& type)
case IPFIX_TYPEID_flowEndNanoSeconds:
return 8;
case IPFIX_TYPEID_bgpSourceAsNumber:
case IPFIX_TYPEID_bgpDestinationAsNumber:
return 0;
default:
THROWEXCEPTION("PacketHashtable: unknown typeid %s, failed to determine raw packet field length", type.toString().c_str());
break;
@ -495,6 +512,7 @@ uint16_t PacketHashtable::getRawPacketFieldOffset(const IeInfo& type, const Pack
{
if (type.enterprise==0 || type.enterprise==IPFIX_PEN_reverse) {
switch (type.id) {
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
return 10;
break;
@ -515,6 +533,7 @@ uint16_t PacketHashtable::getRawPacketFieldOffset(const IeInfo& type, const Pack
break;
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_octetTotalCount:
return 2;
break;
@ -584,6 +603,7 @@ bool PacketHashtable::isRawPacketPtrVariable(const IeInfo& type)
case 0:
case IPFIX_PEN_reverse:
switch (type.id) {
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_flowStartSeconds:
case IPFIX_TYPEID_flowEndSeconds:
@ -592,9 +612,12 @@ bool PacketHashtable::isRawPacketPtrVariable(const IeInfo& type)
case IPFIX_TYPEID_flowStartNanoSeconds: // ^
case IPFIX_TYPEID_flowEndNanoSeconds: // ^
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_sourceIPv4Address:
case IPFIX_TYPEID_destinationIPv4Address:
case IPFIX_TYPEID_bgpSourceAsNumber:
case IPFIX_TYPEID_bgpDestinationAsNumber:
return false;
case IPFIX_TYPEID_icmpTypeCodeIPv4:
@ -703,6 +726,7 @@ bool PacketHashtable::typeAvailable(const IeInfo& type)
switch (type.enterprise) {
case 0:
switch (type.id) {
case IPFIX_TYPEID_packetTotalCount:
case IPFIX_TYPEID_packetDeltaCount:
case IPFIX_TYPEID_flowStartSeconds:
case IPFIX_TYPEID_flowEndSeconds:
@ -710,6 +734,7 @@ bool PacketHashtable::typeAvailable(const IeInfo& type)
case IPFIX_TYPEID_flowEndMilliSeconds:
case IPFIX_TYPEID_flowStartNanoSeconds:
case IPFIX_TYPEID_flowEndNanoSeconds:
case IPFIX_TYPEID_octetTotalCount:
case IPFIX_TYPEID_octetDeltaCount:
case IPFIX_TYPEID_protocolIdentifier:
case IPFIX_TYPEID_sourceIPv4Address:
@ -718,6 +743,8 @@ bool PacketHashtable::typeAvailable(const IeInfo& type)
case IPFIX_TYPEID_sourceTransportPort:
case IPFIX_TYPEID_destinationTransportPort:
case IPFIX_TYPEID_tcpControlBits:
case IPFIX_TYPEID_bgpSourceAsNumber:
case IPFIX_TYPEID_bgpDestinationAsNumber:
return true;
}
break;
@ -1080,10 +1107,12 @@ void PacketHashtable::aggregateField(const ExpFieldData* efd, HashtableBucket* h
#endif
break;
case IPFIX_TYPEID_octetTotalCount: // 8 byte dst, 2 byte src
case IPFIX_TYPEID_octetDeltaCount: // 8 byte dst, 2 byte src
*(uint64_t*)baseData = htonll(ntohll(*(uint64_t*)baseData) + ntohs(*(uint16_t*)deltaData));
break;
case IPFIX_TYPEID_packetTotalCount: // 8 byte dst, no src
case IPFIX_TYPEID_packetDeltaCount: // 8 byte dst, no src
*(uint64_t*)baseData = htonll(ntohll(*(uint64_t*)baseData)+1);
break;
@ -1143,10 +1172,12 @@ void PacketHashtable::aggregateField(const ExpFieldData* efd, HashtableBucket* h
#endif
break;
case IPFIX_TYPEID_octetTotalCount: // 8 byte dst, 2 byte src
case IPFIX_TYPEID_octetDeltaCount: // 8 byte dst, 2 byte src
*(uint64_t*)baseData = htonll(ntohll(*(uint64_t*)baseData) + ntohs(*(uint16_t*)deltaData));
break;
case IPFIX_TYPEID_packetTotalCount: // 8 byte dst, no src
case IPFIX_TYPEID_packetDeltaCount: // 8 byte dst, no src
*(uint64_t*)baseData = htonll(ntohll(*(uint64_t*)baseData)+1);
break;

49
src/modules/ipfix/database/IpfixDbCommon.cpp Normal file
View File

@ -0,0 +1,49 @@
#include "IpfixDbCommon.hpp"
IpfixDbReaderCommonCfg::IpfixDbReaderCommonCfg()
: port(0), timeshift(false), fullspeed(false), observationDomainId(0)
{
}
void IpfixDbReaderCommonCfg::readConfigSection(XMLElement* elem)
{
if (!elem) return;
XMLNode::XMLSet<XMLElement*> set = _elem->getElementChildren();
for (XMLNode::XMLSet<XMLElement*>::iterator it = set.begin();
it != set.end();
it++) {
XMLElement* e = *it;
if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
} else if (e->matches("dbname")) {
dbname = e->getFirstText();
} else if (e->matches("username")) {
user = e->getFirstText();
} else if (e->matches("password")) {
password = e->getFirstText();
} else if (e->matches("timeshift")) {
timeshift = getBool("timeshift", timeshift);
} else if (e->matches("fullspeed")) {
fullspeed = getBool("fullspeed", fullspeed);
} else if (e->matches("observationDomainId")) {
observationDomainId = getInt("observationDomainId");
} else if (e->matches("next")) { // ignore next
} else {
msg(MSG_FATAL, "Unknown IpfixDbReader config statement %s\n", e->getName().c_str());
continue;
}
}
if (hostname=="") THROWEXCEPTION("IpfixDbReaderCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbReaderCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbReaderCfg: dbname not set in configuration!");
if (user=="") THROWEXCEPTION("IpfixDbReaderCfg: username not set in configuration!");
if (password=="") THROWEXCEPTION("IpfixDbReaderCfg: password not set in configuration!");
}

6
src/modules/ipfix/IpfixDbCommon.hpp → src/modules/ipfix/database/IpfixDbCommon.hpp
View File

@ -29,7 +29,9 @@
* EXPORTER_WIDTH : Length of the string for operations on exporter table
*/
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED)
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include "IpfixDbWriterSQL.hpp"
#define STARTLEN 60
#define TABLE_WIDTH 16
@ -61,8 +63,10 @@
#define CN_revLastSwitchedMillis "revLastSwitchedMillis"
#define CN_revTcpControlBits "revTcpControlBits"
#define CN_maxPacketGap "maxPacketGap"
#define CN_revMaxPacketGap "revMaxPacketGap"
#define CN_flowStartSysUpTime "flowStartSysUpTime"
#define CN_flowEndSysUpTime "flowEndSysUpTime"
#endif

194
src/modules/ipfix/database/IpfixDbReader.cpp Normal file
View File

@ -0,0 +1,194 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2008 Gerhard Muenz
* Copyright (C) 2009-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include "IpfixDbReader.hpp"
#include "IpfixDbCommon.hpp"
#include "common/msg.h"
InstanceManager<IpfixTemplateRecord> IpfixDbReader::templateRecordIM("DbReaderIpfixTemplateRecord", 1);
InstanceManager<IpfixDataRecord> IpfixDbReader::dataRecordIM("DbReaderIpfixDataRecord", 50);
InstanceManager<IpfixTemplateDestructionRecord> IpfixDbReader::templateDestructionRecordIM("DbReaderIpfixTemplateDestructionRecord", 1);
/***** Internal Functions ****************************************************/
/**
* First send a a new template, then send the dataTemplates for all stored
* tables.
*/
void* IpfixDbReader::readFromDB(void* ipfixDbReader_)
{
IpfixDbReader* ipfixDbReader = (IpfixDbReader*)ipfixDbReader_;
ipfixDbReader->registerCurrentThread();
if (ipfixDbReader->connectToDb()) {
THROWEXCEPTION("IpfixDbReader creation failed");
}
/** get tables of the database*/
if(ipfixDbReader->getTables() != 0) {
msg(MSG_ERROR,"IpfixDbReader: Error in function getTables");
THROWEXCEPTION("IpfixDbReader creation failed");
}
msg(MSG_DIALOG, "IpfixDbReader: Start sending tables");
for(vector<string>::iterator i = ipfixDbReader->tables.begin(); i != ipfixDbReader->tables.end() && !ipfixDbReader->exitFlag; i++) {
boost::shared_ptr<TemplateInfo> templateInfo(new TemplateInfo);
templateInfo->setId = TemplateInfo::IpfixTemplate;
if(ipfixDbReader->dbReaderSendNewTemplate(templateInfo, *i) != 0)
{
msg(MSG_ERROR, "IpfixDbReader: Template error, skip table");
continue;
}
ipfixDbReader->dbReaderSendTable(templateInfo, *i);
ipfixDbReader->dbReaderDestroyTemplate(templateInfo);
}
ipfixDbReader->unregisterCurrentThread();
msg(MSG_DIALOG,"IpfixDbReader: Sending from database is done");
return 0;
}
/**
* Constructs a template from the table data and sends it to all connected
* modules.
*/
int IpfixDbReader::dbReaderSendNewTemplate(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
// reset record length
recordLength = 0;
/**get columnsname of the table*/
if(getColumns(tableName) != 0) {
msg(MSG_ERROR,"IpfixDbReader: Could not get columns for template");
return 1;
}
for(vector<struct ipfix_identifier>::iterator i = columns.begin(); i != columns.end(); i++) {
templateInfo->fieldCount++;
templateInfo->fieldInfo = (TemplateInfo::FieldInfo*)realloc(templateInfo->fieldInfo,
sizeof(TemplateInfo::FieldInfo)*templateInfo->fieldCount);
TemplateInfo::FieldInfo* fi = &templateInfo->fieldInfo[templateInfo->fieldCount - 1];
fi->type.id = i->id;
fi->type.length = i->length;
fi->type.enterprise = 0;
fi->offset = recordLength;
recordLength = recordLength + fi->type.length;
}
/* Pass Data Template to flowSinks */
IpfixTemplateRecord* ipfixRecord = templateRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReader: sent template for table %s", tableName.c_str());
return 0;
}
void IpfixDbReader::copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type) {
switch (type.length) {
case 1:
*(uint8_t*)dest = *(uint8_t*)src;
return;
case 2:
*(uint16_t*)dest = htons(*(uint16_t*)src);
return;
case 4:
*(uint32_t*)dest = htonl(*(uint32_t*)src);
return;
case 8:
*(uint64_t*)dest = htonll(*(uint64_t*)src);
return;
default:
msg(MSG_ERROR, "IpfixDbReader: Uint with length %d unparseable", type.length);
return;
}
}
int IpfixDbReader::dbReaderDestroyTemplate(boost::shared_ptr<TemplateInfo> templateInfo)
{
IpfixTemplateDestructionRecord* ipfixRecord = templateDestructionRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
send(ipfixRecord);
msg(MSG_DEBUG,"IpfixDbReader: Template destroyed");
return 0;
}
/***** Exported Functions ****************************************************/
/**
* Starts or resumes database
* @param ipfixDbReader handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReader::performStart()
{
thread.run(this);
}
/**
* Temporarily pauses database
* @param ipfixDbReader handle obtained by calling @c createipfixDbReader()
*/
void IpfixDbReader::performShutdown()
{
thread.join();
}
/**
* Frees memory used by an ipfixDbReader
* @param ipfixDbWriter handle obtained by calling @c createipfixDbReader()
*/
IpfixDbReader::~IpfixDbReader() {
}
/**
* Creates a new ipfixDbReader. Do not forget to call @c startipfixDbReader() to begin reading from Database
* @return handle to use when calling @c destroyipfixDbRreader()
*/
IpfixDbReader::IpfixDbReader(const string& dbType, const string& Hostname, const string& Dbname,
const string& Username, const string& Password,
uint16_t Port, uint16_t ObservationDomainId)
: thread(readFromDB), hostname(Hostname), dbname(Dbname), username(Username), password(Password), port(Port), observationDomainId(ObservationDomainId)
{
srcId.reset(new IpfixRecord::SourceID);
srcId->observationDomainId = observationDomainId;
srcId->exporterAddress.len = 0;
srcId->exporterPort = 0;
srcId->receiverPort = 0;
srcId->protocol = 0;
srcId->fileDescriptor = 0;
}
#endif

39
src/modules/ipfix/IpfixDbReader.hpp → src/modules/ipfix/database/IpfixDbReader.hpp
View File

@ -2,6 +2,7 @@
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2006-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -19,14 +20,13 @@
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef DB_SUPPORT_ENABLED
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#ifndef IPFIXDBREADER_H
#define IPFIXDBREADER_H
#include "IpfixRecord.hpp"
#include "common/ipfixlolib/ipfix.h"
#include "../IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include "core/Module.h"
@ -35,8 +35,6 @@
#include <pthread.h>
#include <boost/smart_ptr.hpp>
#include <mysql.h>
/**
* IpfixDbReader powered the communication to the database server
* also between the other structs
@ -44,10 +42,9 @@
class IpfixDbReader : public Module, public Source<IpfixRecord*>, public Destination<NullEmitable*>
{
public:
IpfixDbReader(const string& hostname, const string& dbname,
IpfixDbReader(const string& dbType, const string& hostname, const string& dbname,
const string& username, const string& password,
unsigned port, uint16_t observationDomainId,
bool timeshift, bool fullspeed);
uint16_t port, uint16_t observationDomainId);
~IpfixDbReader();
virtual void performStart();
@ -56,32 +53,34 @@ class IpfixDbReader : public Module, public Source<IpfixRecord*>, public Destina
boost::shared_ptr<IpfixRecord::SourceID> srcId;
protected:
typedef struct {
uint16_t ipfixId; /**IPFIX_TYPEID*/
uint8_t length; /**IPFIX length*/
} columnDB;
void copyUintNetByteOrder(IpfixRecord::Data* dest, char* src, InformationElement::IeInfo type);
vector<string> tables;
vector<columnDB> columns;
vector<struct ipfix_identifier> columns;
string columnNames;
string orderBy;
unsigned recordLength;
bool timeshift, fullspeed;
MYSQL* conn; /** pointer to connection handle */
std::string hostname;
std::string dbname;
std::string username;
std::string password;
uint16_t port;
uint16_t observationDomainId;
Thread thread;
static InstanceManager<IpfixTemplateRecord> templateRecordIM;
static InstanceManager<IpfixDataRecord> dataRecordIM;
static InstanceManager<IpfixTemplateDestructionRecord> templateDestructionRecordIM;
int getTables();
int getColumns(const string& tableName);
static void* readFromDB(void* ipfixDbReader_);
int dbReaderSendNewTemplate(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
int dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
int dbReaderDestroyTemplate(boost::shared_ptr<TemplateInfo> templateInfo);
int connectToDb( const string& hostName, const string& dbName, const string& userName, const string& password, unsigned int port);
virtual int connectToDb() = 0;
virtual int dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName) = 0;
virtual int getColumns(const string& tableName) = 0 ;
virtual int getTables() = 0;
};

45
src/modules/ipfix/IpfixDbReaderCfg.cpp → src/modules/ipfix/database/IpfixDbReaderCfg.cpp
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2009-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -18,10 +18,12 @@
*
*/
#ifdef DB_SUPPORT_ENABLED
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include "IpfixDbReaderCfg.h"
#include "IpfixDbReaderMySQL.hpp"
#include "IpfixDbReaderOracle.hpp"
IpfixDbReaderCfg* IpfixDbReaderCfg::create(XMLElement* e)
{
@ -33,7 +35,7 @@ IpfixDbReaderCfg* IpfixDbReaderCfg::create(XMLElement* e)
IpfixDbReaderCfg::IpfixDbReaderCfg(XMLElement* elem)
: CfgHelper<IpfixDbReader, IpfixDbReaderCfg>(elem, "ipfixDbReader"),
port(0), timeshift(false), fullspeed(false), observationDomainId(0)
port(0), observationDomainId(0)
{
if (!elem) return;
@ -43,7 +45,9 @@ IpfixDbReaderCfg::IpfixDbReaderCfg(XMLElement* elem)
it++) {
XMLElement* e = *it;
if (e->matches("host")) {
if (e->matches("dbType")) {
databaseType = e->getFirstText();
} else if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
@ -53,10 +57,6 @@ IpfixDbReaderCfg::IpfixDbReaderCfg(XMLElement* elem)
user = e->getFirstText();
} else if (e->matches("password")) {
password = e->getFirstText();
} else if (e->matches("timeshift")) {
timeshift = getBool("timeshift", timeshift);
} else if (e->matches("fullspeed")) {
fullspeed = getBool("fullspeed", fullspeed);
} else if (e->matches("observationDomainId")) {
observationDomainId = getInt("observationDomainId");
} else if (e->matches("next")) { // ignore next
@ -65,6 +65,7 @@ IpfixDbReaderCfg::IpfixDbReaderCfg(XMLElement* elem)
continue;
}
}
if (databaseType != "mysql" && databaseType != "postgres" && databaseType != "oracle") THROWEXCEPTION("IpfixDbReaderCfg: Incorrect value for dbType: \"%s\"", databaseType.c_str());
if (hostname=="") THROWEXCEPTION("IpfixDbReaderCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbReaderCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbReaderCfg: dbname not set in configuration!");
@ -80,7 +81,33 @@ IpfixDbReaderCfg::~IpfixDbReaderCfg()
IpfixDbReader* IpfixDbReaderCfg::createInstance()
{
instance = new IpfixDbReader(hostname, dbname, user, password, port, observationDomainId, timeshift, fullspeed);
if (databaseType == "mysql") {
#if defined(DB_SUPPORT_ENABLED)
instance = new IpfixDbReaderMySQL(databaseType, hostname, dbname, user, password, port, observationDomainId);
#else
goto except;
#endif
} else if (databaseType == "postgres") {
#if defined(PG_SUPPORT_ENABLED)
goto except;
#else
goto except;
#endif
} else if (databaseType == "oracle") {
#if defined(ORACLE_SUPPORT_ENABLED)
instance = new IpfixDbReaderOracle(databaseType, hostname, dbname, user, password, port, observationDomainId);
#else
goto except;
#endif
} else {
goto except;
}
return instance;
except:
THROWEXCEPTION("IpfixDbWriterCfg: Database type \"%s\" not yet implemented or support in vermont is not compiled in ...", databaseType.c_str());
// this is only to surpress compiler warnings. we should never get here ...
return 0;
return instance;
}

21
src/modules/ipfix/IpfixDbReaderCfg.h → src/modules/ipfix/database/IpfixDbReaderCfg.h
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2009-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -21,17 +21,15 @@
#ifndef IPFIXDBREADERCFG_H_
#define IPFIXDBREADERCFG_H_
#ifdef DB_SUPPORT_ENABLED
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "modules/ipfix/IpfixDbReader.hpp"
#include "IpfixDbReader.hpp"
#include <string>
using namespace std;
class IpfixDbReaderCfg
: public CfgHelper<IpfixDbReader, IpfixDbReaderCfg>
@ -46,14 +44,13 @@ public:
virtual bool deriveFrom(IpfixDbReaderCfg* old);
protected:
string hostname; /**< hostname of database host */
std::string databaseType; /**< Type of database (mysql, psgl, oracle, ...) */
std::string hostname; /**< hostname of database host */
uint16_t port; /**< port of database */
string dbname; /**< database name */
string user; /**< user name for login to database */
string password; /**< password for login to database */
bool timeshift; /**< shift time stamps */
bool fullspeed; /**< reading in full speed */
std::string dbname; /**< database name */
std::string user; /**< user name for login to database */
std::string password; /**< password for login to database */
std::string type; /**< type of database backend (mysql, oracle, psql) */
uint32_t observationDomainId; /**< observation domain id */
IpfixDbReaderCfg(XMLElement*);

221
src/modules/ipfix/database/IpfixDbReaderMySQL.cpp Normal file
View File

@ -0,0 +1,221 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2008 Gerhard Muenz
* Copyright (C) 2009-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#if defined(DB_SUPPORT_ENABLED)
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include "IpfixDbReaderMySQL.hpp"
#include "IpfixDbCommon.hpp"
#include "common/msg.h"
/***** Internal Functions ****************************************************/
/**
* Select a given table and get the values by reading
* the database. The Typs of the values from database are
* strings, therefore they must change into IPFIX format
*/
int IpfixDbReaderMySQL::dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
unsigned offset = 0;
uint64_t delta = 0; // 64 bit to avoid castings in the case of flowStartMilliseconds
uint32_t flowTime = 0; // in seconds, so 32 bit are sufficient
uint32_t lastFlowTime = 0;
uint64_t tmp;
bool first = true;
unsigned j = 0;
string query = "SELECT " + columnNames + " FROM " + tableName;
msg(MSG_VDEBUG, "IpfixDbReaderMySQL: SQL query: %s", query.c_str());
if(mysql_query(conn, query.c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbReaderMySQL: Select on table failed. Error: %s",
mysql_error(conn));
return 1;
}
dbResult = mysql_store_result(conn);
msg(MSG_INFO,"IpfixDbReaderMySQL: Start sending records from table %s", tableName.c_str());
while((dbRow = mysql_fetch_row(dbResult)) && !exitFlag) {
// build new record
boost::shared_array<IpfixRecord::Data> data(new IpfixRecord::Data[recordLength]);
offset = 0;
j = 0;
for(vector<struct ipfix_identifier>::iterator i = columns.begin(); i != columns.end(); ++i) {
tmp = atoll(dbRow[j]);
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
j++;
}
IpfixDataRecord* ipfixRecord = dataRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
ipfixRecord->dataLength = offset; // = recordLength
ipfixRecord->message = data;
ipfixRecord->data = data.get();
send(ipfixRecord);
msg(MSG_VDEBUG,"IpfixDbReaderMySQL: Record sent");
}
mysql_free_result(dbResult);
if(!exitFlag)
msg(MSG_INFO,"IpfixDbReaderMySQL: Sending from table %s done", tableName.c_str());
else
msg(MSG_INFO,"IpfixDbReaderMySQL: Sending from table %s aborted", tableName.c_str());
return 0;
}
/**
* get all tables in database that matches with the wildcard "h\_%"
**/
int IpfixDbReaderMySQL::getTables()
{
const char* wild = "f\\_%";
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
dbResult = mysql_list_tables(conn, wild);
if(dbResult == 0) {
msg(MSG_FATAL,"IpfixDbReaderMySQL: There are no flow tables in database");
} else {
while((dbRow = mysql_fetch_row(dbResult))) {
tables.push_back(string(dbRow[0]));
msg(MSG_FATAL, "%s", dbRow[0]);
msg(MSG_VDEBUG, "IpfixDbReaderMySQL: table %s", tables.back().c_str());
}
}
mysql_free_result(dbResult);
return 0;
}
/**
* Get the names of columns
*/
int IpfixDbReaderMySQL::getColumns(const string& tableName)
{
MYSQL_RES* dbResult = NULL;
MYSQL_ROW dbRow = NULL;
string query = "SHOW COLUMNS FROM " + tableName;
msg(MSG_VDEBUG, "IpfixDbReaderMySQL: SQL query: %s", query.c_str());
if(mysql_query(conn, query.c_str()) != 0) {
msg(MSG_ERROR,"IpfixDbReaderMySQL: Show columns on table %s failed. Error: %s",
tableName.c_str(), mysql_error(conn));
return 1;
}
dbResult = mysql_store_result(conn);
if(dbResult == 0) {
msg(MSG_FATAL,"IpfixDbReaderMySQL: There are no Columns in the table");
return 1;
}
// TODO: don't we have to free the result of mysql_fetch_row?????
columns.clear();
columnNames = "";
orderBy = "";
while((dbRow = mysql_fetch_row(dbResult))) {
bool found = true;
const struct ipfix_identifier* id = ipfix_name_lookup(dbRow[0]);
if (id == NULL) {
msg(MSG_INFO, "IpfixDbReaderMySQL: Unsupported column: %s", dbRow[0]);
} else {
columnNames = columnNames + "," + dbRow[0];
columns.push_back(*id);
msg(MSG_VDEBUG, "IpfixDbReaderMySQL: column %s (%d)", dbRow[0], columns.back().id);
}
}
// remove leading ","
if(columnNames != "")
columnNames.erase(0,1);
mysql_free_result(dbResult);
return 0;
}
int IpfixDbReaderMySQL::connectToDb()
{
/** get the mysl init handle*/
conn = mysql_init(0);
if(conn == 0) {
msg(MSG_FATAL,"IpfixDbReaderMySQL: Get connect handle failed. Error: %s",
mysql_error(conn));
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReaderMySQL: mysql init successful");
}
/**Connect to Database*/
if (!mysql_real_connect(conn, hostname.c_str(), username.c_str(),password.c_str(),
0, port, 0, 0)) {
msg(MSG_FATAL,"IpfixDbReaderMySQL: Connection to database failed. Error: %s",
mysql_error(conn));
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReaderMySQL: successfully connected to database");
}
/** use database with dbName **/
if(mysql_select_db(conn, dbname.c_str()) !=0) {
msg(MSG_FATAL,"IpfixDbReaderMySQL: Database %s not selectable", dbname.c_str());
return 1;
} else {
msg(MSG_DEBUG,"IpfixDbReaderMySQL: Database %s selected", dbname.c_str());
}
return 0;
}
IpfixDbReaderMySQL::~IpfixDbReaderMySQL() {
mysql_close(conn);
}
/**
* Creates a new ipfixDbReader. Do not forget to call @c startipfixDbReader() to begin reading from Database
* @return handle to use when calling @c destroyipfixDbRreader()
*/
IpfixDbReaderMySQL::IpfixDbReaderMySQL(const string& dbType, const string& Hostname, const string& Dbname,
const string& Username, const string& Password,
uint16_t Port, uint16_t ObservationDomainId)
: IpfixDbReader(dbType, Hostname, Dbname, Username, Password, Port, ObservationDomainId)
{
}
#endif

62
src/modules/ipfix/database/IpfixDbReaderMySQL.hpp Normal file
View File

@ -0,0 +1,62 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2006-2013 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#if defined(DB_SUPPORT_ENABLED)
#ifndef IPFIXDBREADERMYSQL_H
#define IPFIXDBREADERMYSQL_H
#include "IpfixDbReader.hpp"
#include <netinet/in.h>
#include <time.h>
#include <pthread.h>
#include <boost/smart_ptr.hpp>
#include <mysql.h>
/**
* IpfixDbReader powered the communication to the database server
* also between the other structs
*/
class IpfixDbReaderMySQL : public IpfixDbReader
{
public:
IpfixDbReaderMySQL(const string& dbType, const string& hostname, const string& dbname,
const string& username, const string& password,
uint16_t port, uint16_t observationDomainId);
~IpfixDbReaderMySQL();
protected:
MYSQL* conn; /** pointer to connection handle */
virtual int connectToDb();
virtual int dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
virtual int getColumns(const string& tableName);
virtual int getTables();
};
#endif
#endif

284
src/modules/ipfix/database/IpfixDbReaderOracle.cpp Normal file
View File

@ -0,0 +1,284 @@
/*
* IPFIX Database Reader for Oracle DBs
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006-2012 Lothar Braun
* Copyright (C) 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include <sstream>
#include "IpfixDbReaderOracle.hpp"
#include "IpfixDbCommon.hpp"
#include "common/msg.h"
/***** Internal Functions ****************************************************/
int IpfixDbReaderOracle::dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName)
{
std::ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
uint64_t tmp;
bool first = true;
unsigned offset = 0;
unsigned j = 0;
msg(MSG_VDEBUG, "IpfixDbReaderOracle: Sending table %s", tableName.c_str());
sql << "SELECT " << columnNames << " FROM "<< tableName;
// create the oracle statement
try {
stmt = con->createStatement(sql.str());
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbReaderOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbReaderOracle: Table %s was empty!", tableName.c_str());
return 1;
}
msg(MSG_INFO,"IpfixDbReaderOracle: Start sending records from table %s", tableName.c_str());
try {
while((rs->next()) && !exitFlag) {
// build new record
boost::shared_array<IpfixRecord::Data> data(new IpfixRecord::Data[recordLength]);
offset = 0;
j = 0;
for(vector<struct ipfix_identifier>::iterator i = columns.begin(); i != columns.end(); ++i) {
// do not forget: Oracle starts counting at 1
tmp = atoll(rs->getString(j + 1).c_str());
copyUintNetByteOrder(data.get() + templateInfo->fieldInfo[j].offset,
(char*)&tmp,
templateInfo->fieldInfo[j].type);
offset += templateInfo->fieldInfo[j].type.length;
j++;
}
IpfixDataRecord* ipfixRecord = dataRecordIM.getNewInstance();
ipfixRecord->sourceID = srcId;
ipfixRecord->templateInfo = templateInfo;
ipfixRecord->dataLength = offset; // = recordLength
ipfixRecord->message = data;
ipfixRecord->data = data.get();
send(ipfixRecord);
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught SQL exception while getting flows from table: %s", ex.getMessage().c_str());
return 1;
}
if(!exitFlag)
msg(MSG_INFO,"IpfixDbReaderOracle: Sending from table %s done", tableName.c_str());
else
msg(MSG_INFO,"IpfixDbReaderOracle: Sending from table %s aborted", tableName.c_str());
return 0;
}
/**
* get all tables in database that matches with the wildcard "h\_%"
**/
int IpfixDbReaderOracle::getTables()
{
std::ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT table_name FROM user_tables WHERE table_name LIKE 'F_%' ORDER BY table_name ASC";
// create the oracle statement
try {
stmt = con->createStatement(sql.str());
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
//msg(MSG_VDEBUG, "IpfixDbReaderOracle: SQL query: %s", query.c_str());
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Found no flow tables!");
return 1;
}
try {
while((rs->next()) && !exitFlag) {
tables.push_back(rs->getString(1));
msg(MSG_VDEBUG, "IpfixDbReaderOracle: table %s", tables.back().c_str());
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught SQL exception: %s", ex.getMessage().c_str());
return 1;
}
return 0;
}
/**
* Get the names of columns
*/
int IpfixDbReaderOracle::getColumns(const string& tableName)
{
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
string query = "SELECT column_name FROM cols WHERE table_name = '" + tableName + "'";
// create the oracle statement
try {
stmt = con->createStatement(query);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error creating statement: %s", ex.getMessage().c_str());
return 1;
}
//msg(MSG_VDEBUG, "IpfixDbReaderOracle: SQL query: %s", query.c_str());
try {
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing statement: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
}
if (!rs) {
msg(MSG_ERROR, "IpfixDbWriterOracle: Flow tables do not have columns??");
return 1;
}
columns.clear();
columnNames = "";
orderBy = "";
try {
while((rs->next()) && !exitFlag) {
bool found = true;
const struct ipfix_identifier* id = ipfix_name_lookup(rs->getString(1).c_str());
if (id == NULL) {
msg(MSG_INFO, "IpfixDbReaderMySQL: Unsupported column: %s", rs->getString(1).c_str());
} else {
columnNames = columnNames + "," + rs->getString(1).c_str();
columns.push_back(*id);
msg(MSG_VDEBUG, "IpfixDbReaderMySQL: column %s (%d)", rs->getString(1).c_str(), columns.back().id);
}
if(found)
msg(MSG_VDEBUG, "IpfixDbReaderOracle: column %s (%d)", rs->getString(1).c_str(), columns.back().id);
}
if(columnNames != "")
columnNames.erase(0,1);
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_ERROR, "Caught SQL exception: %s", ex.getMessage().c_str());
return 1;
}
return 0;
}
int IpfixDbReaderOracle::connectToDb()
{
dbError = true;
// try to close connection in case we still have one
// from a previous try
if (con) {
env->terminateConnection(con);
}
msg(MSG_DEBUG, "IpfixDbReaderOracle: Creating environment.");
try {
env = oracle::occi::Environment::createEnvironment(oracle::occi::Environment::DEFAULT);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbReaderOracle: Error while creating environment: %s.", ex.getMessage().c_str());
msg(MSG_FATAL, "IpfixDbReaderOracle: Did you configure your Oracle environment?");
return -1;
}
msg(MSG_DEBUG, "IpfixDbReaderOracle: Trying to connect to database ...");
try
{
char dbLogon[256];
sprintf(dbLogon, "%s:%u/%s", hostname.c_str(), port, dbname.c_str());
con = env->createConnection(username, password, dbLogon);
} catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbReaderOracle: Oracle connect failed. Error: %s", ex.getMessage().c_str());
return 1;
}
msg(MSG_DEBUG,"IpfixDbReaderOracle: Successfully connected to Oracle DB");
return 0;
}
/***** Exported Functions ****************************************************/
/**
* Frees memory used by an ipfixDbReader
* @param ipfixDbWriter handle obtained by calling @c createipfixDbReader()
*/
IpfixDbReaderOracle::~IpfixDbReaderOracle() {
env->terminateConnection(con);
oracle::occi::Environment::terminateEnvironment(env);
}
/**
* Creates a new ipfixDbReader. Do not forget to call @c startipfixDbReader() to begin reading from Database
* @return handle to use when calling @c destroyipfixDbRreader()
*/
IpfixDbReaderOracle::IpfixDbReaderOracle(const string& dbType, const string& Hostname, const string& Dbname,
const string& Username, const string& Password,
uint16_t Port, uint16_t ObservationDomainId)
: IpfixDbReader(dbType, Hostname, Dbname, Username, Password, Port, ObservationDomainId)
{
}
#endif

68
src/modules/ipfix/database/IpfixDbReaderOracle.hpp Normal file
View File

@ -0,0 +1,68 @@
/*
* IPFIX Database Reader/Writer for Oracle DBs
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006-2013 Lothar Braun <braun@net.in.tum.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef ORACLE_SUPPORT_ENABLED
#ifndef IPFIXDBREADERORACLE_H
#define IPFIXDBREADERORACLE_H
#include "../IpfixRecord.hpp"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include "core/Module.h"
#include "IpfixDbReader.hpp"
#include <netinet/in.h>
#include <time.h>
#include <pthread.h>
#include <boost/smart_ptr.hpp>
#include <occi.h>
/**
* IpfixDbReader powered the communication to the database server
* also between the other structs
*/
class IpfixDbReaderOracle : public IpfixDbReader
{
public:
IpfixDbReaderOracle(const string& dbType, const string& hostname, const string& dbname,
const string& username, const string& password,
uint16_t port, uint16_t observationDomainId);
~IpfixDbReaderOracle();
protected:
bool dbError; // error flag
oracle::occi::Environment *env;
oracle::occi::Connection *con;
virtual int connectToDb();
virtual int dbReaderSendTable(boost::shared_ptr<TemplateInfo> templateInfo, const string& tableName);
virtual int getColumns(const string& tableName);
virtual int getTables();
};
#endif
#endif

50
src/modules/ipfix/IpfixDbWriterCfg.cpp → src/modules/ipfix/database/IpfixDbWriterCfg.cpp
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2009 - 2012 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -18,9 +18,12 @@
*
*/
#ifdef DB_SUPPORT_ENABLED
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include "IpfixDbWriterCfg.h"
#include "IpfixDbWriterMySQL.hpp"
#include "IpfixDbWriterPg.hpp"
#include "IpfixDbWriterOracle.hpp"
IpfixDbWriterCfg* IpfixDbWriterCfg::create(XMLElement* e)
@ -32,8 +35,8 @@ IpfixDbWriterCfg* IpfixDbWriterCfg::create(XMLElement* e)
IpfixDbWriterCfg::IpfixDbWriterCfg(XMLElement* elem)
: CfgHelper<IpfixDbWriter, IpfixDbWriterCfg>(elem, "ipfixDbWriter"),
port(0), bufferRecords(30), observationDomainId(0)
: CfgHelper<IpfixDbWriterSQL, IpfixDbWriterCfg>(elem, "ipfixDbWriter"),
port(0), bufferRecords(30), observationDomainId(0), useLegacyNames(false)
{
if (!elem) return;
@ -43,7 +46,11 @@ IpfixDbWriterCfg::IpfixDbWriterCfg(XMLElement* elem)
it++) {
XMLElement* e = *it;
if (e->matches("host")) {
if (e->matches("dbType")) {
databaseType = e->getFirstText();
} else if (e->matches("useLegacyNames")) {
useLegacyNames = getBool("useLegacyNames");
} else if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
@ -65,6 +72,7 @@ IpfixDbWriterCfg::IpfixDbWriterCfg(XMLElement* elem)
continue;
}
}
if (databaseType != "mysql" && databaseType != "postgres" && databaseType != "oracle") THROWEXCEPTION("IpfixDbWriterCfg: Incorrect value for dbType: \"%s\"", databaseType.c_str());
if (hostname=="") THROWEXCEPTION("IpfixDbWriterCfg: host not set in configuration!");
if (port==0) THROWEXCEPTION("IpfixDbWriterCfg: port not set in configuration!");
if (dbname=="") THROWEXCEPTION("IpfixDbWriterCfg: dbname not set in configuration!");
@ -94,10 +102,36 @@ IpfixDbWriterCfg::~IpfixDbWriterCfg()
}
IpfixDbWriter* IpfixDbWriterCfg::createInstance()
IpfixDbWriterSQL* IpfixDbWriterCfg::createInstance()
{
instance = new IpfixDbWriter(hostname, dbname, user, password, port, observationDomainId, bufferRecords, colNames);
return instance;
if (databaseType == "mysql") {
#if defined(DB_SUPPORT_ENABLED)
instance = new IpfixDbWriterMySQL(databaseType.c_str(), hostname.c_str(), dbname.c_str(), user.c_str(), password.c_str(), port, observationDomainId, bufferRecords, colNames, useLegacyNames);
#else
goto except;
#endif
} else if (databaseType == "postgres") {
#if defined(PG_SUPPORT_ENABLED)
instance = new IpfixDbWriterPg(databaseType.c_str(), hostname.c_str(), dbname.c_str(), user.c_str(), password.c_str(), port, observationDomainId, bufferRecords, colNames, useLegacyNames);
#else
goto except;
#endif
} else if (databaseType == "oracle") {
#if defined(ORACLE_SUPPORT_ENABLED)
instance = new IpfixDbWriterOracle(databaseType.c_str(), hostname.c_str(), dbname.c_str(), user.c_str(), password.c_str(), port, observationDomainId, bufferRecords, colNames, useLegacyNames);
#else
goto except;
#endif
} else {
goto except;
}
return instance;
except:
THROWEXCEPTION("IpfixDbWriterCfg: Database type \"%s\" not yet implemented or support in vermont is not compiled in ...", databaseType.c_str());
// this is only to surpress compiler warnings. we should never get here ...
return 0;
}

13
src/modules/ipfix/IpfixDbWriterCfg.h → src/modules/ipfix/database/IpfixDbWriterCfg.h
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2009 - 2012 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -21,12 +21,12 @@
#ifndef IPFIXDBWRITERCFG_H_
#define IPFIXDBWRITERCFG_H_
#ifdef DB_SUPPORT_ENABLED
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "modules/ipfix/IpfixDbWriter.hpp"
#include "IpfixDbWriterSQL.hpp"
#include <string>
@ -34,7 +34,7 @@ using namespace std;
class IpfixDbWriterCfg
: public CfgHelper<IpfixDbWriter, IpfixDbWriterCfg>
: public CfgHelper<IpfixDbWriterSQL, IpfixDbWriterCfg>
{
public:
friend class ConfigManager;
@ -42,11 +42,11 @@ public:
virtual IpfixDbWriterCfg* create(XMLElement* e);
virtual ~IpfixDbWriterCfg();
virtual IpfixDbWriter* createInstance();
virtual IpfixDbWriterSQL* createInstance();
virtual bool deriveFrom(IpfixDbWriterCfg* old);
protected:
string databaseType; /**< Type of database (mysql, psgl, oracle, ...) */
string hostname; /**< hostname of database host */
uint16_t port; /**< port of database */
string dbname; /**< database name */
@ -55,6 +55,7 @@ protected:
uint16_t bufferRecords; /**< amount of records to buffer until they are written to database */
uint32_t observationDomainId; /**< default observation domain id (overrides the one received in the records */
vector<string> colNames; /**< column names */
bool useLegacyNames;
void readColumns(XMLElement* elem);
IpfixDbWriterCfg(XMLElement*);

0
src/modules/ipfix/IpfixDbWriterMongo.cpp → src/modules/ipfix/database/IpfixDbWriterMongo.cpp
View File

1
src/modules/ipfix/IpfixDbWriterMongo.hpp → src/modules/ipfix/database/IpfixDbWriterMongo.hpp
View File

@ -28,7 +28,6 @@ Mongo *
#include "IpfixDbCommon.hpp"
#include "IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <iostream>

0
src/modules/ipfix/IpfixDbWriterMongoCfg.cpp → src/modules/ipfix/database/IpfixDbWriterMongoCfg.cpp
View File

2
src/modules/ipfix/IpfixDbWriterMongoCfg.h → src/modules/ipfix/database/IpfixDbWriterMongoCfg.h
View File

@ -26,7 +26,7 @@
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "modules/ipfix/IpfixDbWriterMongo.hpp"
#include "IpfixDbWriterMongo.hpp"
#include <string>

300
src/modules/ipfix/database/IpfixDbWriterMySQL.cpp Normal file
View File

@ -0,0 +1,300 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007, 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriterMySQL and IpfixDbReader */
#ifdef DB_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <iomanip>
#include <stdlib.h>
#include "IpfixDbWriterMySQL.hpp"
#include "common/msg.h"
/**
* (re)connect to database
*/
void IpfixDbWriterMySQL::connectToDB()
{
ostringstream statement;
dbError = true;
// close (in the case that it was already connected)
if (conn) mysql_close(conn);
/** get the mysl init handle*/
conn = mysql_init(0);
if(conn == 0) {
msg(MSG_FATAL,"IpfixDbWriterMySQL: Get MySQL connect handle failed. Error: %s",
mysql_error(conn));
return;
}
msg(MSG_DEBUG,"IpfixDbWriterMySQL: mysql init successful");
/**Connect to Database*/
if (!mysql_real_connect(conn, hostName, userName, password,
0, portNum, 0, 0)) {
msg(MSG_FATAL,"IpfixDbWriterMySQL: Connection to database failed. Error: %s",
mysql_error(conn));
return;
}
msg(MSG_DEBUG,"IpfixDbWriterMySQL: succesfully connected to database");
/** make query string to create database**/
statement << "CREATE DATABASE IF NOT EXISTS " << dbName;
DPRINTF("SQL Query: %s", statement.str().c_str());
/**create database*/
if(mysql_query(conn, statement.str().c_str()) != 0 ) {
msg(MSG_FATAL, "IpfixDbWriterMySQL: Creation of database %s failed. Error: %s",
dbName, mysql_error(conn));
return;
}
msg(MSG_INFO,"IpfixDbWriterMySQL: Database %s created", dbName);
/** use database with dbName**/
if(mysql_select_db(conn, dbName) !=0) {
msg(MSG_FATAL, "IpfixDbWriterMySQL: Database %s not selectable. Error: %s",
dbName, mysql_error(conn));
return ;
}
msg(MSG_DEBUG,"IpfixDbWriterMySQL: Database %s selected", dbName);
if (createExporterTable() != 0) return;
dbError = false;
}
int IpfixDbWriterMySQL::createExporterTable()
{
ostringstream statement;
dbError = true;
/**create table exporter*/
statement.str("");
statement.clear();
statement << "CREATE TABLE IF NOT EXISTS exporter (id SMALLINT(5) NOT NULL AUTO_INCREMENT, sourceID INTEGER(10) UNSIGNED DEFAULT NULL, srcIP INTEGER(10) UNSIGNED DEFAULT NULL, PRIMARY KEY(id))";
DPRINTF("SQL Query: %s", statement.str().c_str());
if(mysql_query(conn, statement.str().c_str()) != 0) {
msg(MSG_FATAL,"IpfixDbWriterMySQL: Creation of exporter table failed. Error: %s",
mysql_error(conn));
return 1;
}
msg(MSG_INFO,"IpfixDbWriterMySQL: Exporter table created");
dbError = false;
return 0;
}
/**
* Create a table in the database
*/
bool IpfixDbWriterMySQL::createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime)
{
uint32_t i;
if (find(usedPartitions.begin(), usedPartitions.end(), partitionname)!=usedPartitions.end()) {
// found cached entry!
DPRINTF("Partition '%s' already created.", partitionname);
return true;
}
ostringstream ctsql;
ostringstream oss;
ctsql << "CREATE TABLE IF NOT EXISTS " << partitionname << " (";
/**collect the names for columns and the dataTypes for the table in a string*/
for(i=0; i < numberOfColumns; i++) {
ctsql << tableColumns[i].cname << " " << tableColumns[i].dataType;
if (i != numberOfColumns-1) {
ctsql << ", ";
}
}
ctsql << ")";
msg(MSG_FATAL, "SQL Query: %s", ctsql.str().c_str());
if(mysql_query(conn, ctsql.str().c_str()) != 0) {
msg(MSG_FATAL,"IpfixDbWriterMySQL: Creation of flow table failed. Error: %s",
mysql_error(conn));
dbError = true;
return 1;
}
msg(MSG_INFO, "Partition %s created ", partitionname);
usedPartitions.push_back(partitionname);
if (usedPartitions.size()>MAX_USEDTABLES) usedPartitions.pop_front();
string indexname = string(partitionname) + "_firstswitched";
ostringstream cisql;
cisql << "CREATE INDEX " << indexname <<" ON " << partitionname;
cisql << "(firstswitched)";
if(mysql_query(conn, ctsql.str().c_str()) != 0) {
msg(MSG_FATAL,"IpfixDbWriterMySQL: Creation of index failed. Error: %s",
mysql_error(conn));
dbError = true;
return true;
}
msg(MSG_INFO, "Index %s_firstswitched created ", partitionname);
return true;
}
/**
* loop over table columns and template to get the IPFIX values in correct order to store in database
* The result is written into row, the firstSwitched time is returned in flowstartsec
*/
/*
* Write insertStatement to database
*/
bool IpfixDbWriterMySQL::writeToDb()
{
if (insertBuffer.curRows == 0) return true;
DPRINTF("SQL Query: %s", insertBuffer.sql);
if(mysql_query(conn, insertBuffer.sql) != 0) {
msg(MSG_ERROR,"IpfixDbWriterMySQL: Insert of records failed. Error: %s", mysql_error(conn));
goto dbwriteerror;
}
insertBuffer.curRows = 0;
insertBuffer.appendPtr = insertBuffer.bodyPtr;
*insertBuffer.appendPtr = 0;
msg(MSG_DEBUG,"IpfixDbWriterMySQL: Write to database is complete");
return true;
dbwriteerror:
dbError = true;
return false;
}
/**
* Returns the exporterID
* For every different sourcID and expIp a unique ExporterID will be generated from the database
* First lookup for the ExporterID in the exporterBuffer according sourceID and expIp, is there nothing
* lookup in the ExporterTable, is there also nothing insert sourceID and expIp an return the generated
* ExporterID
*/
int IpfixDbWriterMySQL::getExporterID(IpfixRecord::SourceID* sourceID)
{
uint32_t i;
int exporterID = 0;
MYSQL_RES* dbResult;
MYSQL_ROW dbRow;
char statementStr[EXPORTER_WIDTH];
uint32_t expIp = 0;
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID->exporterAddress.toUInt32();
/** Is the exporterID already in exporterBuffer? */
for(i = 0; i < curExporterEntries; i++) {
if(exporterEntries[i].observationDomainId == sourceID->observationDomainId &&
exporterEntries[i].ip==expIp) {
DPRINTF("Exporter sourceID/IP with ID %d is in the exporterBuffer\n",
exporterEntries[i].Id);
return exporterEntries[i].Id;
}
}
// it is not: try to get it from the database
sprintf(statementStr, "SELECT id FROM exporter WHERE sourceID=%u AND srcIp='%s'", sourceID->observationDomainId, IPToString(expIp).c_str());
if(mysql_query(conn, statementStr) != 0) {
msg(MSG_ERROR,"IpfixDbWriterMySQL: Select on exporter table failed. Error: %s",
mysql_error(conn));
dbError = true;
return 0;// If a failure occurs, return 0
}
dbResult = mysql_store_result(conn);
if(( dbRow = mysql_fetch_row(dbResult))) {
// found in table
exporterID = atoi(dbRow[0]);
mysql_free_result(dbResult);
DPRINTF("ExporterID %d is in exporter table", exporterID);
} else {
mysql_free_result(dbResult);
// insert new exporter table entry
/**ExporterID is not in exporter table - insert expID and expIp and return the exporterID*/
sprintf(statementStr, "INSERT INTO exporter (sourceID, srcIP) VALUES ('%u','%s')",
sourceID->observationDomainId, IPToString(expIp).c_str());
if(mysql_query(conn, statementStr) != 0) {
msg(MSG_ERROR,"IpfixDbWriterMySQL: Insert in exporter table failed. Error: %s", conn);
dbError = true;
return 0;
}
exporterID = mysql_insert_id(conn);
msg(MSG_INFO,"IpfixDbWriterMySQL: new exporter (ODID=%d, id=%d) inserted in exporter table", sourceID->observationDomainId, exporterID);
}
if (curExporterEntries==MAX_EXP_TABLE-1) {
// maybe here we should check how often this happens and display a severe warning if too
// many parallel streams are received at once
msg(MSG_INFO, "IpfixDbWriterPg: turnover for exporter cache occurred.");
curExporterEntries = 0;
}
/**Write new exporter in the exporterBuffer*/
exporterEntries[curExporterEntries].Id = exporterID;
exporterEntries[curExporterEntries].observationDomainId = sourceID->observationDomainId;
exporterEntries[curExporterEntries++].ip = expIp;
return exporterID;
}
/***** Public Methods ****************************************************/
IpfixDbWriterMySQL::IpfixDbWriterMySQL(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId,
int maxStatements, vector<string> columns, bool legacyNames)
: IpfixDbWriterSQL(dbType, host, db, user, pw, port, observationDomainId, maxStatements, columns, legacyNames), conn(0)
{
connectToDB();
}
IpfixDbWriterMySQL::~IpfixDbWriterMySQL()
{
writeToDb();
if (conn) mysql_close(conn);
}
#endif

69
src/modules/ipfix/database/IpfixDbWriterMySQL.hpp Normal file
View File

@ -0,0 +1,69 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007, 2008 Gerhard Muenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriterMySQL and IpfixDbReader */
#ifdef DB_SUPPORT_ENABLED
#ifndef IPFIXDBWRITER_H
#define IPFIXDBWRITER_H
#include "IpfixDbCommon.hpp"
#include "IpfixDbWriterSQL.hpp"
#include "../IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <mysql.h>
#include <netinet/in.h>
#include <time.h>
#include <sstream>
#define EXPORTERID 0
/**
* IpfixDbWriterMySQL powered the communication to the database server
* also between the other structs
*/
class IpfixDbWriterMySQL
: public IpfixDbWriterSQL
{
public:
IpfixDbWriterMySQL(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId, // FIXME: observationDomainId
int maxStatements, vector<string> columns, bool useLegacyNames);
~IpfixDbWriterMySQL();
virtual void connectToDB();
virtual bool writeToDb();
virtual int createExporterTable();
virtual int getExporterID(IpfixRecord::SourceID* sourceID);
virtual bool createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime);
private:
MYSQL* conn;
};
#endif
#endif

575
src/modules/ipfix/database/IpfixDbWriterOracle.cpp Normal file
View File

@ -0,0 +1,575 @@
/*
* IPFIX Database Reader/Writer Oracle Connector
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef ORACLE_SUPPORT_ENABLED
#include <stdexcept>
#include <string.h>
#include <iomanip>
#include <stdlib.h>
#include "IpfixDbWriterOracle.hpp"
#include "common/msg.h"
/**
* (re)connect to database
*/
void IpfixDbWriterOracle::connectToDB()
{
dbError = true;
// close (in the case that it was already connected)
if (con) env->terminateConnection(con);
/** get the initial environment and connect */
msg(MSG_DEBUG, "IpfixDbWriterOracle: Creating environment.");
try {
env = oracle::occi::Environment::createEnvironment(oracle::occi::Environment::DEFAULT);
} catch (oracle::occi::SQLException& ex) {
msg(MSG_FATAL, "IpfixDbWriterOracle: Error while creating environment: %s.", ex.getMessage().c_str());
msg(MSG_FATAL, "IpfixDbWriterOracle: Did you configure your Oracle environment?");
return ;
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Trying to connect to database ...");
try
{
char dbLogon[256];
sprintf(dbLogon, "%s:%u/%s", hostName, portNum, dbName);
con = env->createConnection(userName, password, dbLogon);
} catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Oracle connect failed. Error: %s", ex.getMessage().c_str());
return ;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: Oracle connection successful");
if (createExporterTable()!=0) return ;
dbError = false;
}
std::string IpfixDbWriterOracle::insertRowPrefix()
{
ostringstream sql;
sql << "INTO " << curTable.name << " (";
for (uint32_t i = 0; i < numberOfColumns; ++i) {
sql << tableColumns[i].cname;
if (i < numberOfColumns - 1) sql << ",";
}
sql << ") VALUES ";
return sql.str();
}
int IpfixDbWriterOracle::createExporterTable()
{
// check if table exists
ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT COUNT(table_name) FROM user_tables WHERE table_name='EXPORTER'";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing create exporter table: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
if (rs)
{
while(rs->next())
{
if (rs->getInt(1)!= 0)
{
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table does exist");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
return 0;
}
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
// create table
sql.str("");
sql << "CREATE TABLE exporter ( id NUMERIC(10) NOT NULL, sourceID NUMERIC(10), srcIP NUMERIC(10), CONSTRAINT exporter_pk PRIMARY KEY (id) )";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating exporter table statement: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating exporter table: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
// create counter
// clear vars for reuse
sql.str("");
sql << "CREATE sequence counter_for_exporter increment BY 1 start WITH 1 cache 2";
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating sequence counter statement: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating squence counter table: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table counter created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
// create trigger
sql.str("");
sql << "CREATE OR REPLACE TRIGGER trigger_for_id_exporter BEFORE INSERT ON exporter REFERENCING NEW AS new FOR EACH ROW Begin SELECT counter_for_exporter.NEXTVAL INTO :new.id FROM DUAL; End;";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing trigger creation \"%s\": %s", sql.str().c_str(), ex.getMessage().c_str());
dbError = true;
con->terminateStatement(stmt);
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table insert trigger created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
msg(MSG_DEBUG, "Exporter table creation done");
return 0;
}
/*
* Write insertStatement to database
*/
bool IpfixDbWriterOracle::writeToDb()
{
if (insertBuffer.curRows == 0) {
// nothing to write
return 1;
}
// this is an insert operation. Oracle is a professional datatbase and therefore wants to have
// some special handling of multi row inserts. Inserts should look like
// INSERT ALL
// INTO <tablename> (<column-spec>) VALUES (<values>)
// INTO <tablenaem> (<column-spec>) VALUES (<values>)
// ...
// SELECT * FROM dual;
// we therefore need to append the dual select
std::string sql_string = std::string(insertBuffer.sql) + " SELECT * FROM dual";
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
try
{
stmt = con->createStatement(sql_string.c_str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error creating statement: %s", ex.getMessage().c_str());
dbError = true;
return 0;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Error executing flow db insert \"%s\": %s", insertBuffer.sql, ex.getMessage().c_str());
dbError = true;
con->terminateStatement(stmt);
return 0;
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
insertBuffer.curRows = 0;
insertBuffer.appendPtr = insertBuffer.bodyPtr;
*insertBuffer.appendPtr = 0;
// commit transaction
try {
con->commit();
} catch (oracle::occi::SQLException& ex) {
dbError = true;
msg(MSG_FATAL, "IpfixDbWriterOracle: Received exception during commit: \"%s\"", ex.getMessage().c_str());
}
return 1;
}
bool IpfixDbWriterOracle::createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime)
{
uint32_t i;
if (find(usedPartitions.begin(), usedPartitions.end(), partitionname)!=usedPartitions.end()) {
// found cached entry!
DPRINTF("Partition '%s' already created.", partitionname);
return true;
}
// check if table exists
ostringstream sql;
oracle::occi::Statement *stmt = NULL;
oracle::occi::ResultSet *rs = NULL;
sql << "SELECT COUNT(table_name) FROM user_tables WHERE table_name='" << partitionname<< "'";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
if (rs)
{
while(rs->next())
{
if (rs->getInt(1)!= 0)
{
msg(MSG_DEBUG,"IpfixDbWriterOracle: table does exist");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
return 0;
}
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
// create table
sql.str("");
sql << "CREATE TABLE " << partitionname<< " ( " << tableColumnsCreateString << ")";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Failed to prepare CREATE flow table statement \"%s: %s", sql.str().c_str(), ex.getMessage().c_str());
dbError = true;
return 1;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Failed to execute CREATE flow statement \"%s\": %s", sql.str().c_str(), ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 1;
}
msg(MSG_DEBUG,"IpfixDbWriterOracle: exporter table created");
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
msg(MSG_DEBUG, "IpfixDbWriterOracle: Table %s created ", partitionname);
return 0;
}
string IpfixDbWriterOracle::getInsertString(string tableName)
{
// oracle multi-row insert statement differs from mysql/postgres multirow insert
return "INSERT ALL ";
}
/**
* Returns the id of the exporter table entry or 0 in the case of an error
*/
int IpfixDbWriterOracle::getExporterID(IpfixRecord::SourceID* sourceID)
{
uint32_t i;
oracle::occi::Statement* stmt = NULL;
oracle::occi::ResultSet* rs = NULL;
int exporterID = -1;
uint32_t expIp = 0;
ostringstream sql;
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID->exporterAddress.toUInt32();
/** Is the exporterID already in exporterBuffer? */
for(i = 0; i < curExporterEntries; i++) {
if(exporterEntries[i].observationDomainId == sourceID->observationDomainId &&
exporterEntries[i].ip==expIp) {
DPRINTF("Exporter sourceID/IP with ID %d is in the exporterBuffer\n",
exporterEntries[i].Id);
return exporterEntries[i].Id;
}
}
// search exporter table
sql << "SELECT id FROM exporter WHERE sourceID=" << sourceID->observationDomainId << " AND srcIp=" << expIp;
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on exporter table failed. Error: %s", ex.getMessage().c_str());
return 0;// If a failure occurs, return 0
}
if(stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
if (rs)
{
while(rs->next())
{
exporterID = rs->getInt(1);
msg(MSG_DEBUG, "IpfixDbWriterOracle: ExporterID %d is in exporter table", exporterID);
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on exporter table failed. Error: %s", ex.getMessage().c_str());
con->terminateStatement(stmt);
dbError = true;
return 0;// If a failure occurs, return 0
}
}
// insert new entry in exporter table since it is not found
if(exporterID == -1)
{
sql.str("");
sql << "INSERT INTO exporter (sourceID,srcIP) VALUES ('" << sourceID->observationDomainId << "','" << expIp << "')";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Insert in exporter table failed. Error: %s", ex.getMessage().c_str());
dbError = true;
return 0;
}
if (stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
}
catch (oracle::occi::SQLException& ex)
{
msg(MSG_FATAL,"IpfixDbWriterOracle: Insert in exporter table failed. Error: %s", ex.getMessage().c_str());
dbError = true;
con->terminateStatement(stmt);
return 0;
}
stmt->closeResultSet(rs);
con->terminateStatement(stmt);
}
sql.str("");
sql << "SELECT counter_for_exporter.CURRVAL FROM DUAL";
msg(MSG_DEBUG, "IpfixDbWriterOracle: SQL Query: %s", sql.str().c_str());
try
{
stmt = con->createStatement(sql.str());
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on counter_for_exporter sequence failed. Error: %s", ex.getMessage().c_str());
dbError = true;
return 0;// If a failure occurs, return 0
}
if(stmt)
{
try
{
stmt->setPrefetchRowCount(1);
rs = stmt->executeQuery();
if (rs)
{
while(rs->next())
{
exporterID = rs->getInt(1);
DPRINTF("ExporterID %d is in exporter table", exporterID);
}
stmt->closeResultSet(rs);
}
con->terminateStatement(stmt);
}
catch (oracle::occi::SQLException &ex)
{
msg(MSG_ERROR,"IpfixDbWriterOracle: Select on counter_for_exporter sequence failed. Error: %s", ex.getMessage().c_str());
dbError = true;
con->terminateStatement(stmt);
return 0;// If a failure occurs, return 0
}
msg(MSG_INFO,"IpfixDbWriterOracle: new exporter (ODID=%d, id=%d) inserted in exporter table", sourceID->observationDomainId, exporterID);
}
}
if (curExporterEntries==MAX_EXP_TABLE-1) {
// maybe here we should check how often this happens and display a severe warning if too
// many parallel streams are received at once
msg(MSG_INFO, "IpfixDbWriterPg: turnover for exporter cache occurred.");
curExporterEntries = 0;
}
/**Write new exporter in the exporterBuffer*/
exporterEntries[curExporterEntries].Id = exporterID;
exporterEntries[curExporterEntries].observationDomainId = sourceID->observationDomainId;
exporterEntries[curExporterEntries++].ip = expIp;
return exporterID;
}
/***** Public Methods ****************************************************/
IpfixDbWriterOracle::IpfixDbWriterOracle(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId,
int maxStatements, vector<string> columns, bool legacyNames)
: IpfixDbWriterSQL(dbType, host, db, user, pw, port, observationDomainId, maxStatements, columns, legacyNames), con(0), env(0)
{
connectToDB();
}
/**
* Destructor
*/
IpfixDbWriterOracle::~IpfixDbWriterOracle()
{
writeToDb();
env->terminateConnection(con);
oracle::occi::Environment::terminateEnvironment(env);
}
#endif /* ORACLE_SUPPORT_ENABLED */

76
src/modules/ipfix/database/IpfixDbWriterOracle.hpp Normal file
View File

@ -0,0 +1,76 @@
/*
* IPFIX Database Reader/Writer Oracle Connector
* Copyright (C) 2011 Philipp Fehre <philipp.fehre@googlemail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef ORACLE_SUPPORT_ENABLED
#ifndef IPFIXDBWRITERORACLE_H_
#define IPFIXDBWRITERORACLE_H_
#include "IpfixDbCommon.hpp"
#include "IpfixDbWriterSQL.hpp"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <occi.h>
#include <iostream>
#include <iomanip>
#include <stdlib.h>
#include <stdio.h>
#include <netinet/in.h>
#include <time.h>
#include <sstream>
using namespace std;
#define EXPORTERID 0
/**
* IpfixDbWriterMySQL powered the communication to the database server
* also between the other structs
*/
class IpfixDbWriterOracle
: public IpfixDbWriterSQL
{
public:
IpfixDbWriterOracle(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId, // FIXME: observationDomainId
int maxStatements, vector<string> columns, bool legacyNames);
~IpfixDbWriterOracle();
virtual void connectToDB();
virtual bool writeToDb();
virtual int createExporterTable();
virtual int getExporterID(IpfixRecord::SourceID* sourceID);
virtual bool createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime);
virtual string getInsertString(string tableName);
virtual string insertRowPrefix();
private:
oracle::occi::Environment *env;
oracle::occi::Connection *con;
};
#endif
#endif

352
src/modules/ipfix/database/IpfixDbWriterPg.cpp Normal file
View File

@ -0,0 +1,352 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007 Gerhard Muenz
* Copyright (C) 2008 Tobias Limmer
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriterPg and IpfixDbReader */
#ifdef PG_SUPPORT_ENABLED
#include "IpfixDbWriterPg.hpp"
#include "common/msg.h"
#include "common/Misc.h"
#include "common/Time.h"
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include <sstream>
#include <algorithm>
using namespace std;
/***** Global Variables ******************************************************/
/**
* maximum length of one item in a SQL statement
*/
const uint16_t MAX_COL_LENGTH = 22;
/**
* (re)connect to database
*/
void IpfixDbWriterPg::connectToDB()
{
dbError = true;
// close (in the case that it was already connected)
if (conn) PQfinish(conn);
/**Connect to Database*/
ostringstream conninfo;
conninfo << "host='" << hostName << "' port='" << portNum << "' ";
conninfo << "dbname='" << dbName << "' user='" << userName<< "' ";
conninfo << "password='" << password << "'";// sslmode=require";
DPRINTF("using connection string '%s'", conninfo.str().c_str());
conn = PQconnectdb(conninfo.str().c_str());
if (PQstatus(conn) != CONNECTION_OK) {
msg(MSG_FATAL, "IpfixDbWriterPg: Connection to database failed, error: %s", PQerrorMessage(conn));
return;
}
/**create table exporter*/
if (createExporterTable()!=0) return;
dbError = false;
}
int IpfixDbWriterPg::createExporterTable()
{
/** create table exporter*/
ostringstream oss;
oss << "SELECT COUNT(*) FROM pg_class where relname='exporter'";
PGresult* res = PQexec(conn, oss.str().c_str());
DPRINTF("PQntuples: %d", PQntuples(res));
if((PQresultStatus(res) != PGRES_TUPLES_OK) || (PQntuples(res)==0)) {
msg(MSG_FATAL, "IpfixDbWriterPg: Failed to check if table 'exporter' exists. Error: %s",
PQerrorMessage(conn));
PQclear(res);
dbError = true;
return 1;
}
if (atoi(PQgetvalue(res, 0, 0))!=1) {
string ctexporter = "CREATE TABLE exporter (id serial PRIMARY KEY, "
"sourceID integer, "
"srcIP inet)";
res = PQexec(conn, ctexporter.c_str());
if(PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL, "IpfixDbWriterPg: Creation of table Exporter failed. Error: %s",
PQerrorMessage(conn));
PQclear(res);
dbError = true;
return 1;
} else {
PQclear(res);
msg(MSG_DEBUG, "Exporter table created");
}
}
return 0;
}
/**
* Create a table in the database
*/
bool IpfixDbWriterPg::createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime)
{
uint32_t i;
if (find(usedPartitions.begin(), usedPartitions.end(), partitionname)!=usedPartitions.end()) {
// found cached entry!
DPRINTF("Partition '%s' already created.", partitionname);
return true;
}
ostringstream ctsql;
ostringstream oss;
// check if table needs to be created
if (!checkRelationExists(tablePrefix.c_str())) {
ctsql << "CREATE TABLE " << tablePrefix << " (";
/**collect the names for columns and the dataTypes for the table in a string*/
for(i=0; i < numberOfColumns; i++) {
ctsql << tableColumns[i].cname << " " << tableColumns[i].dataType;
if (i != numberOfColumns-1) {
ctsql << ", ";
}
}
ctsql << ")";
/** create table*/
PGresult* res = PQexec(conn, ctsql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of table failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Table %s created ", tablePrefix.c_str());
}
}
if (!checkRelationExists(partitionname)) {
// create partition
ostringstream cpsql;
cpsql << "CREATE TABLE " << partitionname << " (CHECK (firstswitched>='";
//cpsql << getTimeAsString(starttime, "%Y-%m-%d %H:%M:%S", true);
cpsql << starttime;
cpsql << "' AND firstswitched<'" << endtime; //getTimeAsString(endtime, "%Y-%m-%d %H:%M:%S", true);
cpsql << "')) INHERITS (" << tablePrefix << ")";
PGresult* res = PQexec(conn, cpsql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of partition failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Partition %s created ", partitionname);
usedPartitions.push_back(partitionname);
if (usedPartitions.size()>MAX_USEDTABLES) usedPartitions.pop_front();
}
}
string indexname = string(partitionname) + "_firstswitched";
if (!checkRelationExists(indexname.c_str())) {
ostringstream cisql;
cisql << "CREATE INDEX " << indexname <<" ON " << partitionname;
cisql << "(firstswitched)";
PGresult* res = PQexec(conn, cisql.str().c_str());
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_FATAL,"IpfixDbWriterPg: Creation of index failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
PQclear(res);
return false;
} else {
PQclear(res);
msg(MSG_INFO, "Index %s_firstswitched created ", partitionname);
}
}
return true;
}
/**
* Function writes the content of the statemBuffer to database
* statemBuffer consist of single insert statements
*/
bool IpfixDbWriterPg::writeToDb()
{
if (insertBuffer.curRows==0) return true;
// Write rows to database
msg(MSG_FATAL, "%s", insertBuffer.sql);
PGresult* res = PQexec(conn, insertBuffer.sql);
if (PQresultStatus(res) != PGRES_COMMAND_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Insert of records failed. Error: %s",
PQerrorMessage(conn));
PQclear(res);
goto dbwriteerror;
}
PQclear(res);
insertBuffer.curRows = 0;
insertBuffer.appendPtr = insertBuffer.bodyPtr;
*insertBuffer.appendPtr = 0;
msg(MSG_DEBUG,"Write to database is complete");
return true;
dbwriteerror:
dbError = true;
return false;
}
/**
* Returns the exporterID
* For every different sourcID and expIp a unique ExporterID will be generated from the database
* First lookup for the ExporterID in the exporterBuffer according sourceID and expIp, is there nothing
* lookup in the ExporterTable, is there also nothing insert sourceID and expIp an return the generated
* ExporterID
*/
int IpfixDbWriterPg::getExporterID(IpfixRecord::SourceID* sourceID)
{
uint32_t i;
int exporterID = 0;
char statementStr[EXPORTER_WIDTH];
uint32_t expIp = 0;
// convert IP address (correct host byte order since 07/2010)
expIp = sourceID->exporterAddress.toUInt32();
/** Is the exporterID already in exporterBuffer? */
for(i = 0; i < curExporterEntries; i++) {
if(exporterEntries[i].observationDomainId == sourceID->observationDomainId &&
exporterEntries[i].ip==expIp) {
DPRINTF("Exporter sourceID/IP with ID %d is in the exporterBuffer\n",
exporterEntries[i].Id);
return exporterEntries[i].Id;
}
}
// it is not: try to get it from the database
sprintf(statementStr, "SELECT id FROM exporter WHERE sourceID=%u AND srcIp='%s'", sourceID->observationDomainId, IPToString(expIp).c_str());
PGresult* res = PQexec(conn, statementStr);
if (PQresultStatus(res) != PGRES_TUPLES_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Select on exporter table failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
return 0;// If a failure occurs, return exporterID = 0
}
/** is the exporterID in the exporter table ?*/
if (PQntuples(res)) {
exporterID = atoi(PQgetvalue(res, 0, 0));
DPRINTF("ExporterID %d is in exporter table\n",exporterID);
}
else
{
PQclear(res);
/**ExporterID is not in exporter table - insert expID and expIp and return the exporterID*/
sprintf(statementStr, "INSERT INTO exporter (sourceID, srcIP) VALUES ('%u','%s') RETURNING id",
sourceID->observationDomainId, IPToString(expIp).c_str());
res = PQexec(conn, statementStr);
if(PQresultStatus(res) != PGRES_TUPLES_OK) {
msg(MSG_ERROR,"IpfixDbWriterPg: Insert in exporter table failed. Error: %s",
PQerrorMessage(conn));
dbError = true;
return 0;
}
exporterID = atoi(PQgetvalue(res, 0, 0));
msg(MSG_INFO,"ExporterID %d inserted in exporter table", exporterID);
}
PQclear(res);
if (curExporterEntries==MAX_EXP_TABLE-1) {
// maybe here we should check how often this happens and display a severe warning if too
// many parallel streams are received at once
msg(MSG_INFO, "IpfixDbWriterPg: turnover for exporter cache occurred.");
curExporterEntries = 0;
}
/**Write new exporter in the exporterBuffer*/
exporterEntries[curExporterEntries].Id = exporterID;
exporterEntries[curExporterEntries].observationDomainId = sourceID->observationDomainId;
exporterEntries[curExporterEntries++].ip = expIp;
return exporterID;
}
bool IpfixDbWriterPg::checkRelationExists(const char* relname)
{
// check if table needs to be created
ostringstream oss;
oss << "SELECT COUNT(*) FROM pg_class where relname='" << relname << "'";
PGresult* res = PQexec(conn, oss.str().c_str());
if((PQresultStatus(res) != PGRES_TUPLES_OK) || (PQntuples(res)==0)) {
msg(MSG_FATAL, "IpfixDbWriterPg: Failed to check if relation '%s' exists. Error: %s",
relname, PQerrorMessage(conn));
PQclear(res);
dbError = true;
return false;
}
if (atoi(PQgetvalue(res, 0, 0))==1) {
PQclear(res);
return true;
}
PQclear(res);
return false;
}
/***** Exported Functions ****************************************************/
IpfixDbWriterPg::IpfixDbWriterPg(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId,
int maxStatements, vector<string> columns, bool legacyNames)
: IpfixDbWriterSQL(dbType, host, db, user, pw, port, observationDomainId, maxStatements, columns, legacyNames), conn(0)
{
connectToDB();
}
IpfixDbWriterPg::~IpfixDbWriterPg()
{
writeToDb();
if (conn) PQfinish(conn);
}
#endif

71
src/modules/ipfix/database/IpfixDbWriterPg.hpp Normal file
View File

@ -0,0 +1,71 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007 Gerhard Muenz
* Copyright (C) 2008 Tobias Limmer
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef PG_SUPPORT_ENABLED
#ifndef IPFIXDBWRITERPG_H
#define IPFIXDBWRITERPG_H
#include "IpfixDbCommon.hpp"
#include "IpfixDbWriterSQL.hpp"
#include "../IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <libpq-fe.h>
#include <netinet/in.h>
#include <time.h>
#define EXPORTERID 0
/**
* IpfixDbWriterPg powered the communication to the database server
* also between the other structs
*/
class IpfixDbWriterPg
: public IpfixDbWriterSQL
{
public:
IpfixDbWriterPg(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId, // FIXME: observationDomainId
int maxStatements, vector<string> columns, bool legacyNames);
~IpfixDbWriterPg();
virtual void connectToDB();
virtual bool writeToDb();
virtual int createExporterTable();
virtual int getExporterID(IpfixRecord::SourceID* sourceID);
virtual bool createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime);
protected:
PGconn* conn;
bool checkRelationExists(const char* relname);
};
#endif
#endif

752
src/modules/ipfix/database/IpfixDbWriterSQL.cpp Normal file
View File

@ -0,0 +1,752 @@
/*
* IPFIX Database Writer Base for SQL-based Databases
* Copyright (C) 2012 Lothar Braun
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
/* Some constants that are common to IpfixDbWriterSQL and IpfixDbReader */
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include "IpfixDbWriterSQL.hpp"
#include "common/msg.h"
#include "common/Misc.h"
#include "common/Time.h"
#include <stdexcept>
#include <string.h>
#include <stdlib.h>
#include <sstream>
#include <algorithm>
using namespace std;
#define EXPORTERID 0
/***** Structs for SQL databases *********************************************/
const static IpfixDbWriterSQL::Column legacyNames [] = {
{ CN_dstIP, IPFIX_TYPEID_destinationIPv4Address, "", 0, 0},
{ CN_srcIP, IPFIX_TYPEID_sourceIPv4Address, "", 0, 0},
{ CN_srcPort, IPFIX_TYPEID_sourceTransportPort, "", 0, 0},
{ CN_dstPort, IPFIX_TYPEID_destinationTransportPort, "", 0, 0},
{ CN_proto, IPFIX_TYPEID_protocolIdentifier, "", 0, 0 },
{ CN_dstTos, IPFIX_TYPEID_classOfServiceIPv4, "", 0, 0},
{ CN_bytes, IPFIX_TYPEID_octetDeltaCount, "", 0, 0},
{ CN_pkts, IPFIX_TYPEID_packetDeltaCount, "", 0, 0},
{ CN_firstSwitched, IPFIX_TYPEID_flowStartMilliSeconds, "", 0, 0}, // default value is invalid/not used for this ent
{ CN_lastSwitched, IPFIX_TYPEID_flowEndMilliSeconds, "", 0, 0}, // default value is invalid/not used for this entry
{ CN_tcpControlBits, IPFIX_TYPEID_tcpControlBits, "", 0, 0},
//TODO: use enterprise number for the following extended types (Gerhard, 12/2009)
{ CN_revbytes, IPFIX_TYPEID_octetDeltaCount, "", IPFIX_PEN_reverse, 0},
{ CN_revpkts, IPFIX_TYPEID_packetDeltaCount, "", IPFIX_PEN_reverse, 0},
{ CN_revFirstSwitched, IPFIX_TYPEID_flowStartMilliSeconds, "", IPFIX_PEN_reverse, 0}, // default value is invalid/not used for this entry
{ CN_revLastSwitched, IPFIX_TYPEID_flowEndMilliSeconds, "", IPFIX_PEN_reverse, 0}, // default value is invalid/not used for this entry
{ CN_revTcpControlBits, IPFIX_TYPEID_tcpControlBits, "", IPFIX_PEN_reverse, 0},
{ CN_maxPacketGap, IPFIX_ETYPEID_maxPacketGap, "", IPFIX_PEN_vermont|IPFIX_PEN_reverse, 0},
{ CN_exporterID, EXPORTERID, "", 0, 0},
{ CN_flowStartSysUpTime, IPFIX_TYPEID_flowStartSysUpTime, "", 0, 0},
{ CN_flowEndSysUpTime, IPFIX_TYPEID_flowEndSysUpTime, "", 0, 0},
{ "", 0, "", 0, 0 } // last entry must be 0
};
/***** Global Variables ******************************************************/
/**
* maximum length of one item in a SQL statement
*/
const uint16_t MAX_COL_LENGTH = 22;
/****** Methods **************************************************************/
std::string IpfixDbWriterSQL::getDBDataType(uint16_t ipfixTypeLength)
{
if (dbType == "mysql") {
switch (ipfixTypeLength) {
case 1:
return "TINYINT UNSIGNED";
case 2:
return "SMALLINT UNSIGNED";
case 4:
return "INT UNSIGNED";
case 8:
return "BIGINT UNSIGNED";
case 65535:
// variable length, we only support fields up to 100 bytes (be careful, this may waste a lot of diskspace ...")
return "VARCHAR(100)";
default:
THROWEXCEPTION("IpfixDbWriter: Type with non matching length %d ", ipfixTypeLength);
}
} else if (dbType == "postgres") {
// TODO: postgres does not do unsigned types. we therefore use the bigger field. this wastes
/// disk space. Optimize! (except bigints ...)
switch (ipfixTypeLength) {
case 1:
return "smallint";
case 2:
return "integer";
case 4:
return "bigint";
case 8:
return "bigint";
case 65535:
// variable length, we only support fields up to 100 bytes (be careful, this may waste a lot of diskspace ...")
return "VARCHAR(100)";
default:
THROWEXCEPTION("IpfixDbWriter: Type with non matching length %d ", ipfixTypeLength);
}
} else if (dbType == "oracle") {
// TODO: someone should think about proper type lengths ...
switch (ipfixTypeLength) {
case 1:
return "NUMBER(3)";
case 2:
return "NUMBER(6)";
case 4:
return "NUMBER(10)";
case 8:
return "NUMBER(20)";
case 65535:
// variable length, we only support fields up to 100 bytes (be careful, this may waste a lot of diskspace ...")
return "VARCHAR(100)";
default:
THROWEXCEPTION("IpfixDbWriter: Type with non matching length %d ", ipfixTypeLength);
}
}
THROWEXCEPTION("IpfixDbWriter: Found unsupported database backend \"%s\". This is a programming error! Please contact vermont-dev@berlios.de!", dbType.c_str());
// make compiler happy. we should never get here
return "";
}
/**
* save given elements of record to database
*/
void IpfixDbWriterSQL::processDataDataRecord(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data)
{
DPRINTF("Processing data record");
if (dbError) {
connectToDB();
if (dbError) return;
}
/** check if statement buffer is not full*/
if(insertBuffer.curRows==insertBuffer.maxRows) {
msg(MSG_ERROR, "failed to write data to database, trying again ...");
if (!writeToDb()) {
msg(MSG_ERROR, "dropping record");
return;
}
}
/** sourceid null ? use default*/
/* overwrite sourceid if defined */
if(srcId.observationDomainId != 0 || sourceID == NULL) {
sourceID = &srcId;
}
// insert record into buffer
fillInsertRow(sourceID, dataTemplateInfo, length, data);
// statemBuffer is filled -> insert in table
if(insertBuffer.curRows==insertBuffer.maxRows) {
msg(MSG_INFO, "Writing buffered records to database");
writeToDb();
}
}
/**
* function receive the when callback is started
*/
void IpfixDbWriterSQL::onDataRecord(IpfixDataRecord* record)
{
// only treat non-Options Data Records (although we cannot be sure that there is a Flow inside)
if((record->templateInfo->setId != TemplateInfo::NetflowTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixDataTemplate)) {
record->removeReference();
return;
}
processDataDataRecord(record->sourceID.get(), record->templateInfo.get(),
record->dataLength, record->data);
record->removeReference();
}
bool IpfixDbWriterSQL::checkCurrentTable(uint64_t flowStart)
{
return curTable.timeStart!=0 && (curTable.timeStart<=flowStart && curTable.timeEnd>flowStart);
}
string IpfixDbWriterSQL::getTimeAsString(uint64_t milliseconds, const char* formatstring, bool addfraction, uint32_t microseconds)
{
char timebuffer[40];
struct tm date;
time_t seconds = milliseconds/1000; // round down to start of interval and convert ms -> s
gmtime_r(&seconds, &date);
strftime(timebuffer, ARRAY_SIZE(timebuffer), formatstring, &date);
if (addfraction) {
#if DEBUG
if (ARRAY_SIZE(timebuffer)-strlen(timebuffer)<7) THROWEXCEPTION("IpfixDbWriterSQL: buffer size too small");
#endif
snprintf(timebuffer+strlen(timebuffer), 8, ".%06u", static_cast<uint32_t>(milliseconds%1000)*1000+(microseconds%1000));
}
return string(timebuffer);
}
bool IpfixDbWriterSQL::setCurrentTable(uint64_t flowStart)
{
if (insertBuffer.curRows) THROWEXCEPTION("programming error: setCurrentTable MUST NOT be called when entries are still cached!");
string tablename = tablePrefix;
uint64_t starttime = (flowStart/TABLE_INTERVAL)*TABLE_INTERVAL; // round down to start of interval
uint64_t endtime = starttime+TABLE_INTERVAL;
tablename += getTimeAsString(starttime, "_%y%m%d_%H%M%S", false);
if (!createDBTable(tablename.c_str(), starttime, endtime)) return false;
string sql = getInsertString(tablename);
strcpy(insertBuffer.sql, sql.c_str());
insertBuffer.bodyPtr = insertBuffer.sql + sql.size();
insertBuffer.appendPtr = insertBuffer.bodyPtr;
curTable.name = tablename;
curTable.timeStart = starttime;
curTable.timeEnd = endtime;
return true;
}
// extract seconds, ms and ys from ntp time
void IpfixDbWriterSQL::extractNtp64(uint64_t& intdata, uint32_t& micros)
{
if (intdata==0) {
micros = 0;
return;
}
timeval t = timentp64(*((ntp64*)(&intdata)));
intdata = (uint64_t)t.tv_sec*1000+t.tv_usec/1000;
micros = t.tv_usec%1000;
}
std::string IpfixDbWriterSQL::insertRowPrefix()
{
if (insertBuffer.curRows > 0) {
return ",";
}
return "";
}
/**
* loop over the TemplateInfo (fieldinfo,datainfo) to get the IPFIX values to store in database
* results are stored in insertBuffer.sql
*/
void IpfixDbWriterSQL::fillInsertRow(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length, IpfixRecord::Data* data)
{
uint64_t intdata = 0;
uint64_t flowstart = 0;
uint64_t intdata2 = 0;
uint32_t k;
bool notfound, notfound2;
bool first = true;
ostringstream rowStream;
rowStream << "(";
/**loop over the columname and loop over the IPFIX_TYPEID of the record
to get the corresponding data to store and make insert statement*/
for(vector<Column>::iterator col = tableColumns.begin(); col != tableColumns.end(); col++) {
if (col->ipfixId == EXPORTERID) {
intdata = (uint64_t)getExporterID(sourceID);
} else {
notfound = true;
// try to gather data required for the field
if(dataTemplateInfo->fieldCount > 0) {
// look inside the ipfix record
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo->fieldInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
DPRINTF("IpfixDbWriter::getdata: really saw ipfix id %d (%s) in packet with intdata %llX, type %d, length %d and offset %X", col->ipfixId, ipfix_id_lookup(col->ipfixId, col->enterprise)->name, intdata, dataTemplateInfo->fieldInfo[k].type.id, dataTemplateInfo->fieldInfo[k].type.length, dataTemplateInfo->fieldInfo[k].offset);
break;
}
}
}
if( dataTemplateInfo->dataCount > 0 && notfound) {
// look in static data fields of template for data
for(k=0; k < dataTemplateInfo->dataCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type.enterprise == col->enterprise && dataTemplateInfo->dataInfo[k].type.id == col->ipfixId) {
notfound = false;
intdata = getdata(dataTemplateInfo->dataInfo[k].type,(dataTemplateInfo->data+dataTemplateInfo->dataInfo[k].offset));
break;
}
}
}
if(notfound) {
notfound2 = true;
// for some Ids, we have an alternative
if(col->enterprise == 0) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
// look for alternative (flowStartMilliSeconds/1000)
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowStartMilliSeconds) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow start time is available, maybe this is is from a netflow from Cisco
// then - as a last alternative - use flowStartSysUpTime as flow start time
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSysUpTime) {
intdata2 = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
// look for alternative (flowStartSeconds*1000)
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSeconds) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
}
// if no flow start time is available, maybe this is is from a netflow from Cisco
// then - as a last alternative - use flowStartSysUpTime as flow start time
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowStartSysUpTime) {
intdata2 = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
// look for alternative (flowEndMilliSeconds/1000)
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowEndMilliSeconds) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
// if no flow end time is available, maybe this is is from a netflow from Cisco
// then use flowEndSysUpTime as flow start time
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSysUpTime) {
intdata2 = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset));
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
// look for alternative (flowEndSeconds*1000)
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSeconds) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
}
// if no flow end time is available, maybe this is is from a netflow from Cisco
// then use flowEndSysUpTime as flow start time
if(dataTemplateInfo->fieldInfo[k].type.id == IPFIX_TYPEID_flowEndSysUpTime) {
intdata2 = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound2 = false;
}
}
if(notfound && !notfound2) {
intdata = intdata2;
notfound = false;
}
}
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// look for alternative (revFlowStartMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
// look for alternative (revFlowStartSeconds*1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowStartSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndSeconds:
// look for alternative (revFlowEndMilliSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndMilliSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) / 1000;
notfound = false;
break;
}
}
}
break;
case IPFIX_TYPEID_flowEndMilliSeconds:
// look for alternative (revFlowEndSeconds/1000)
if(dataTemplateInfo->fieldCount > 0) {
for(k=0; k < dataTemplateInfo->fieldCount; k++) {
if(dataTemplateInfo->fieldInfo[k].type == InformationElement::IeInfo(IPFIX_TYPEID_flowEndSeconds, IPFIX_PEN_reverse)) {
intdata = getdata(dataTemplateInfo->fieldInfo[k].type,(data+dataTemplateInfo->fieldInfo[k].offset)) * 1000;
notfound = false;
break;
}
}
}
}
}
// if still not found, get default value
if(notfound)
intdata = col->defaultValue;
}
// we need extra treatment for timing related fields
if(col->enterprise == 0 ) {
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartSeconds:
// save time for table access
if (flowstart==0) flowstart = intdata * 1000;
break;
case IPFIX_TYPEID_flowEndSeconds:
break;
case IPFIX_TYPEID_flowStartMilliSeconds:
// if flowStartSeconds is not stored in one of the columns, but flowStartMilliSeconds is,
// then we use flowStartMilliSeconds for table access
// This is realized by storing this value only if flowStartSeconds has not yet been seen.
// A later appearing flowStartSeconds will override this value.
if (flowstart==0)
flowstart = intdata;
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
//intdata %= 1000;
break;
}
} else if (col->enterprise==IPFIX_PEN_reverse)
switch (col->ipfixId) {
case IPFIX_TYPEID_flowStartMilliSeconds:
case IPFIX_TYPEID_flowEndMilliSeconds:
// in the database the millisecond entry is counted from last second
//intdata %= 1000;
break;
}
}
DPRINTF("saw ipfix id %d in packet with intdata %llX", col->ipfixId, intdata);
if(first)
rowStream << intdata;
else
rowStream << "," << intdata;
first = false;
}
rowStream << ")";
// if this flow belongs to a different table, flush all cached entries now
// and get new table
if (!checkCurrentTable(flowstart)) {
if (insertBuffer.curRows != 0 && !writeToDb()) {
msg(MSG_ERROR, "failed to flush table, dropping record");
return;
}
if (!setCurrentTable(flowstart)) {
msg(MSG_ERROR, "failed to change table, dropping record");
return;
}
}
// add prefix that needs to be applied before the actual values for
// the insert string
std::string prefix= insertRowPrefix();
if (prefix.size() > 0) {
strcat(insertBuffer.appendPtr, prefix.c_str());
insertBuffer.appendPtr += prefix.size();
}
// add the rowStream buffer
strcat(insertBuffer.appendPtr, rowStream.str().c_str());
insertBuffer.appendPtr += rowStream.str().size();
insertBuffer.curRows++;
}
/**
* Get data of the record is given by the IPFIX_TYPEID
*/
uint64_t IpfixDbWriterSQL::getdata(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
if(type.id == IPFIX_TYPEID_sourceIPv4Address || type.id == IPFIX_TYPEID_destinationIPv4Address) {
return getipv4address(type, data);
} else {
return getIPFIXValue(type, data);
}
}
/**
* determine the ipv4address of the data record
*/
uint32_t IpfixDbWriterSQL::getipv4address( InformationElement::IeInfo type, IpfixRecord::Data* data)
{
if (type.length > 5) {
DPRINTF("IPv4 Address with length %d unparseable\n", type.length);
return 0;
}
if ((type.length == 5) && ( type.id == IPFIX_TYPEID_sourceIPv4Address || IPFIX_TYPEID_destinationIPv4Address )) /*&& (imask != 0)*/{
DPRINTF("imask drop from ipaddress\n");
type.length = 4;
}
if ((type.length < 5) &&( type.id == IPFIX_TYPEID_sourceIPv4Address || type.id == IPFIX_TYPEID_destinationIPv4Address)) /*&& (imask == 0)*/{
return getIPFIXValue(type, data);
}
return 0;
}
/**
* get the IPFIX value
*/
uint64_t IpfixDbWriterSQL::getIPFIXValue(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
switch (type.length) {
case 1:
return (*(uint8_t*)data);
case 2:
return ntohs(*(uint16_t*)data);
case 4:
return ntohl(*(uint32_t*)data);
case 8:
return ntohll(*(uint64_t*)data);
default:
printf("Uint with length %d unparseable\n", type.length);
return 0;
}
}
/**
* if there no IPFIX_TYPEID in the given data record
* get the default value to store in the database columns
*/
uint32_t IpfixDbWriterSQL::getdefaultIPFIXdata(int ipfixtype_id)
{
int i;
for( i=0; i != tableColumns.size(); i++) {
if(ipfixtype_id == tableColumns[i].ipfixId) {
return tableColumns[i].defaultValue;
}
}
return 0;
}
string IpfixDbWriterSQL::getInsertString(string tableName)
{
ostringstream sql;
sql << "INSERT INTO " << tableName << " (";
for (uint32_t i = 0; i < numberOfColumns; ++i) {
sql << tableColumns[i].cname;
if (i < numberOfColumns - 1) sql << ",";
}
sql << ") VALUES ";
return sql.str();
}
/***** Exported Functions ****************************************************/
/**
* Creates a new IpfixDbWriterSQL. Do not forget to call @c startipfixDbWriter() to begin writing to Database
* @return handle to use when calling @c destroyipfixDbWriter()
*/
IpfixDbWriterSQL::IpfixDbWriterSQL(const char* dbtype, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId,
int maxStatements, vector<string> columns, bool legacyNames)
{
/**Initialize structure members IpfixDbWriterSQL*/
hostName = host;
dbName = db;
userName = user;
password = pw;
portNum = port;
dbType = dbtype;
socketName = 0;
useLegacyNames = legacyNames;
flags = 0;
srcId.exporterAddress.len = 0;
srcId.observationDomainId = observationDomainId;
srcId.exporterPort = 0;
srcId.receiverPort = 0;
srcId.protocol = 0;
srcId.fileDescriptor = 0;
bzero(&exporterEntries, sizeof(exporterEntries));
curExporterEntries = 0;
curTable.timeStart = 0;
curTable.timeEnd = 0;
curTable.name = "";
tablePrefix = "f"; // TODO: make this in config file configurable!
legacyNamesMap = (Column*)::legacyNames;
if(columns.empty())
THROWEXCEPTION("IpfixDbWriter: cannot initiate with no columns");
/* get columns */
bool first = true;
for(vector<string>::const_iterator col = columns.begin(); col != columns.end(); col++) {
std::string columnName = *col;
std::string dataType = "";
uint16_t ipfixId;
uint32_t enterpriseId;
if (useLegacyNames) {
bool found = false;
int i = 0;
while(legacyNamesMap[i].cname != "") {
if(col->compare(legacyNamesMap[i].cname) == 0) {
found = true;
columnName = legacyNamesMap[i].cname;
ipfixId = legacyNamesMap[i].ipfixId;
enterpriseId = legacyNamesMap[i].enterprise;
const struct ipfix_identifier* identifier = ipfix_id_lookup(ipfixId, enterpriseId);
// special handling for ht exporter field
if (ipfixId == 0) {
dataType = getDBDataType(2);
} else {
if (NULL == identifier) {
THROWEXCEPTION("Programming error: Legacy Type name \"%s (id: %u)\" does not have a entry in ipfixlolib!", columnName.c_str(), ipfixId);
}
dataType = getDBDataType(identifier->length);
}
}
i++;
}
if (!found) {
THROWEXCEPTION("IpfixDbWriter: useLegacyNames was activated but name \"%s\" does not match any legacy name!", col->c_str());
}
} else {
const struct ipfix_identifier* identifier = ipfix_name_lookup(col->c_str());
if (NULL == identifier) {
// only identifier that is not part of the standard IPFIX or VERMONT IPFIX name space is
// the exporterID field. We need to handle this manuallly.
if (col->compare(CN_exporterID) == 0) {
ipfixId = EXPORTERID;
enterpriseId = 0;
dataType = getDBDataType(2);
} else {
THROWEXCEPTION("IpfixDbWriter: Could not find a matching IPFIX type for \"%s\". Cannot map the IPFIX type in IPFIX messages to the column names. If you think VERMONT should support type \"%s\", please contact the developers at vermont-dev@berlios.de", col->c_str(), col->c_str());
}
} else {
ipfixId = identifier->id;
enterpriseId = identifier->pen;
dataType = getDBDataType(identifier->length);
}
}
Column c;
c.cname = columnName;
c.ipfixId = ipfixId;
c.enterprise = enterpriseId;
c.dataType = dataType;
c.defaultValue = 0;
tableColumns.push_back(c);
// update tableColumnsString
if(!first)
tableColumnsString.append(",");
tableColumnsString.append(c.cname);
// update tableColumnsCreateString
if(!first)
tableColumnsCreateString.append(", ");
tableColumnsCreateString.append(c.cname);
tableColumnsCreateString.append(" ");
tableColumnsCreateString.append(c.dataType);
first = false;
}
msg(MSG_INFO, "IpfixDbWriter: columns are %s", tableColumnsString.c_str());
/**count columns*/
numberOfColumns = tableColumns.size();
/**Initialize structure members Statement*/
insertBuffer.curRows = 0;
insertBuffer.maxRows = maxStatements;
insertBuffer.sql = new char[(INS_WIDTH+3)*(numberOfColumns+1)*maxStatements+numberOfColumns*20+60+1];
*insertBuffer.sql = 0;
}
/**
* Frees memory used by an IpfixDbWriterSQL
* @param IpfixDbWriterSQL handle obtained by calling @c createipfixDbWriter()
*/
IpfixDbWriterSQL::~IpfixDbWriterSQL()
{
delete[] insertBuffer.sql;
}
#endif

96
src/modules/ipfix/IpfixDbWriterPg.hpp → src/modules/ipfix/database/IpfixDbWriterSQL.hpp
View File

@ -1,9 +1,6 @@
/*
* IPFIX Database Reader/Writer
* Copyright (C) 2006 Jürgen Abberger
* Copyright (C) 2006 Lothar Braun <braunl@informatik.uni-tuebingen.de>
* Copyright (C) 2007 Gerhard Muenz
* Copyright (C) 2008 Tobias Limmer
* IPFIX Database Base Class for SQL based databases
* Copyright (C) 2012 Lothar Braun
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -22,45 +19,60 @@
*/
/* Some constants that are common to IpfixDbWriter and IpfixDbReader */
#ifdef PG_SUPPORT_ENABLED
#ifndef _IPFIX_DB_WRITER_SQL_H_
#define _IPFIX_DB_WRITER_SQL_H_
#ifndef IPFIXDBWRITERPG_H
#define IPFIXDBWRITERPG_H
#if defined(DB_SUPPORT_ENABLED) || defined(MONGO_SUPPORT_ENABLED) || defined(PG_SUPPORT_ENABLED) || defined(ORACLE_SUPPORT_ENABLED) || defined(REDIS_SUPPORT_ENABLED)
#include "IpfixDbCommon.hpp"
#include "IpfixRecordDestination.h"
#include "../IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <libpq-fe.h>
#include <netinet/in.h>
#include <time.h>
#define EXPORTERID 0
/**
* IpfixDbWriterPg powered the communication to the database server
* also between the other structs
*/
class IpfixDbWriterPg
class IpfixDbWriterSQL
: public IpfixRecordDestination, public Module, public Source<NullEmitable*>
{
public:
IpfixDbWriterPg(const char* host, const char* db,
IpfixDbWriterSQL(const char* dbType, const char* host, const char* db,
const char* user, const char* pw,
unsigned int port, uint16_t observationDomainId, // FIXME: observationDomainId
int maxStatements);
~IpfixDbWriterPg();
int maxStatements, vector<string> columns, bool useLegacyNames);
~IpfixDbWriterSQL();
void onDataRecord(IpfixDataRecord* record);
IpfixRecord::SourceID srcId; /**Exporter default SourceID */
/**
* Identify the depency between columns names and
* IPFIX_TYPEID working with a char pointer array
* in this array there is also standing the defaultvalue
* of the IPFIX_TYPEID and the datatype to store in database
*/
struct Column {
std::string cname; /** column name */
uint16_t ipfixId; /** IPFIX_TYPEID */
std::string dataType; /** which datatype to store in database */
uint32_t enterprise;
/**
* when no IPFIX_TYPEID is stored in the record,
* use defaultvalue to store in database
*/
int defaultValue;
};
protected:
static const uint32_t MAX_EXP_TABLE = 10; /**< Count of buffered exporters. Increase this value if you use more exporters in parallel */
static const uint32_t MAX_USEDTABLES = 5; /**< Number of cached entries for used (and created tables) */
static const uint64_t TABLE_INTERVAL = 1000*24*3600; /**< Interval, in which new tables should be created (milliseconds).*/
/**
* Buffer for insert statements
*/
@ -103,25 +115,36 @@ class IpfixDbWriterPg
unsigned int portNum; /** Portnumber (use default) */
const char* socketName; /** Socketname (use default) */
unsigned int flags; /** Connectionflags (none) */
PGconn* conn; /** pointer to connection handle */
int dbError;
Table curTable; /** table name for currently cached entries in insertBuffer */
string tablePrefix; /** prefix for all tables */
string dbType;
bool useLegacyNames;
vector<Column> tableColumns; // table columns
string tableColumnsString; // table columns as string for INSERT statements
string tableColumnsCreateString; // table create string
int createExporterTable();
bool createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime);
void addColumnEntry(const char* insert, bool quoted, bool lastcolumn);
void addColumnEntry(const uint64_t insert, bool quoted, bool lastcolumn);
void fillInsertRow(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length, IpfixRecord::Data* data);
bool writeToDb();
int getExporterID(IpfixRecord::SourceID* sourceID);
bool checkCurrentTable(uint64_t flowStart);
bool setCurrentTable(uint64_t flowStart);
string getTimeAsString(uint64_t milliseconds, const char* formatstring, bool addfraction, uint32_t microseconds = 0);
bool checkCurrentTable(uint64_t flowStart);
bool setCurrentTable(uint64_t flowStart);
string getTimeAsString(uint64_t milliseconds, const char* formatstring, bool addfraction, uint32_t microseconds = 0);
bool checkRelationExists(const char* relname);
virtual void connectToDB() = 0;
virtual bool writeToDb() = 0;
virtual int createExporterTable() = 0 ;
//virtual string createInsertStatement() = 0;
virtual bool createDBTable(const char* partitionname, uint64_t starttime, uint64_t endtime) = 0;
virtual string getInsertString(string tableName);
virtual int getExporterID(IpfixRecord::SourceID* sourceID) = 0;
virtual string insertRowPrefix();
std::string getDBDataType(uint16_t ipfixTypeLength);
Column* legacyNamesMap;
private:
void connectToDB();
void processDataDataRecord(IpfixRecord::SourceID* sourceID,
TemplateInfo* dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data);
@ -137,29 +160,8 @@ class IpfixDbWriterPg
uint32_t getipv4address(InformationElement::IeInfo type, IpfixRecord::Data* data);
void extractNtp64(uint64_t& intdata, uint32_t& micros);
/**
* Identify the depency between columns names and
* IPFIX_TYPEID working with a char pointer array
* in this array there is also standing the defaultvalue
* of the IPFIX_TYPEID and the datatype to store in database
*/
struct Column {
const char* cname; /** column name */
uint16_t ipfixId; /** IPFIX_TYPEID */
const char* dataType; /** which datatype to store in database */
uint32_t enterprise;
/**
* when no IPFIX_TYPEID is stored in the record,
* use defaultvalue to store in database
*/
int defaultValue;
};
const static Column identify[];
};
#endif
#endif

235
src/modules/ipfix/database/IpfixFlowInspectorExporter.cpp Normal file
View File

@ -0,0 +1,235 @@
/*
* IPFIX Database Writer Redis Connector
* Copyright (C) 2012 Lothar Braun <braun@net.in.tum.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef REDIS_SUPPORT_ENABLED
#include <string.h>
#include <iomanip>
#include <stdlib.h>
#include <vector>
#include "IpfixFlowInspectorExporter.hpp"
#include "common/msg.h"
#include <hiredis/hiredis.h>
/**
* (re)connect to database
*/
int IpfixFlowInspectorExporter::connectToDB()
{
dbError = true;
// If a connection exists don't reconnect
if (context) return 0;
// Connect
std::string err;
context = redisConnect(dbHost.c_str(), dbPort);
if (context->err) {
msg(MSG_FATAL,"IpfixFlowInspectorExporter: Redis connect failed. Error: %s", context->errstr);
redisFree(context);
context = NULL;
return 1;
}
msg(MSG_DEBUG,"IpfixFlowInspectorExporter: Connection to Redis successful");
dbError = false;
return 0;
}
/**
* save record to database
*/
void IpfixFlowInspectorExporter::processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data)
{
std::string json_string;
msg(MSG_DEBUG, "IpfixFlowInspectorExporter: Processing data record");
if (dbError) {
msg(MSG_DEBUG, "IpfixFlowInspectorExporter: reconnecting to DB");
connectToDB();
if (dbError) return;
}
json_string = getInsertObj(dataTemplateInfo, length, data);
bufferedObjects.push_back(json_string);
writeToDb();
}
std::string IpfixFlowInspectorExporter::getInsertObj(TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data)
{
std::stringstream sstream;
const struct ipfix_identifier* identifier;
uint64_t intdata;
sstream << "{ ";
/* Dump all elements to DB */
if(dataTemplateInfo.fieldCount > 0) {
// look in ipfix records
for(int k=0; k < dataTemplateInfo.fieldCount; k++) {
intdata = getData(dataTemplateInfo.fieldInfo[k].type,(data+dataTemplateInfo.fieldInfo[k].offset));
DPRINTF("IpfixFlowInspectorExporter::getData: dumping from packet intdata %llX, type %d, length %d and offset %X",
intdata, dataTemplateInfo.fieldInfo[k].type.id, dataTemplateInfo.fieldInfo[k].type.length,
dataTemplateInfo.fieldInfo[k].offset);
if (k != 0) {
sstream << ",";
}
identifier = ipfix_id_lookup(dataTemplateInfo.fieldInfo[k].type.id, dataTemplateInfo.fieldInfo[k].type.enterprise);
if (identifier) {
// push regular IPFIX name into queue
sstream << "\"" << identifier->name << "\" : ";
} else {
sstream << dataTemplateInfo.fieldInfo[k].type.id;
}
sstream << static_cast<long long int>(intdata);
}
}
if( dataTemplateInfo.dataCount > 0) {
// look in static data fields of template for data
for(int k=0; k < dataTemplateInfo.dataCount; k++) {
intdata = getData(dataTemplateInfo.dataInfo[k].type,(dataTemplateInfo.data+dataTemplateInfo.dataInfo[k].offset));
if (k != 0) {
sstream << ",";
}
identifier = ipfix_id_lookup(dataTemplateInfo.dataInfo[k].type.id, dataTemplateInfo.dataInfo[k].type.enterprise);
if (identifier) {
// push regular IPFIX name into queue
sstream << "\"" << identifier->name << "\" : ";
} else {
sstream << dataTemplateInfo.dataInfo[k].type.id;
}
sstream << static_cast<long long int>(intdata);
}
}
sstream << " }";
return sstream.str();
}
/*
* Write Objects to database
*/
int IpfixFlowInspectorExporter::writeToDb()
{
while (!bufferedObjects.empty()) {
std::string& elem = bufferedObjects.front();
redisReply *reply;
reply = (redisReply*)redisCommand(context, "RPUSH %s %s", dbName.c_str(), elem.c_str());
if (!reply) {
msg(MSG_ERROR, "IpfixFlowInspectorExporter: Error while writing to redis queue: %s", reply->str);
freeReplyObject(reply);
}
bufferedObjects.pop_front();
}
return 0;
}
/**
* Get data of the record is given by the IPFIX_TYPEID
*/
uint64_t IpfixFlowInspectorExporter::getData(InformationElement::IeInfo type, IpfixRecord::Data* data)
{
// TODO: workout a proper modular interpreter
if (type.id >= IPFIX_TYPEID_mplsLabelStackEntry1 &&
type.id <= IPFIX_TYPEID_mplsLabelStackEntry10) {
// Properly decode MPLS Label, Experimental and S fields
// TODO: Provide all separated fields
uint32_t MPLSStruct = 0;
uint32_t Label = 0, Exp = 0, S = 0;
memcpy(&MPLSStruct, data, 3);
Label = MPLSStruct & 0xfffff0;
Exp = MPLSStruct & 0x00000e;
S = MPLSStruct & 0x000001;
return ntohl(Label);
}
switch (type.length) {
case 1:
return (*(uint8_t*)data);
case 2:
return ntohs(*(uint16_t*)data);
case 4:
return ntohl(*(uint32_t*)data);
case 5: // may occur in the case if IP address + mask
return ntohl(*(uint32_t*)data);
case 8:
return ntohll(*(uint64_t*)data);
default:
// TODO: dynamic lenght octectArray fields
printf("Got element %d with unparsable length(%d).\n", type.id, type.length);
return 0;
}
}
/***** Public Methods ****************************************************/
/**
* called on Data Record arrival
*/
void IpfixFlowInspectorExporter::onDataRecord(IpfixDataRecord* record)
{
// only treat non-Options Data Records (although we cannot be sure that there is a Flow inside)
if((record->templateInfo->setId != TemplateInfo::NetflowTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixTemplate)
&& (record->templateInfo->setId != TemplateInfo::IpfixDataTemplate)) {
record->removeReference();
return;
}
msg(MSG_DEBUG, "IpfixFlowInspectorExporter: Data record received will be passed for processing");
processDataDataRecord(*record->sourceID.get(), *record->templateInfo.get(),
record->dataLength, record->data);
record->removeReference();
}
/**
* Constructor
*/
IpfixFlowInspectorExporter::IpfixFlowInspectorExporter(const string& hostname, const string& database, unsigned port)
: dbHost(hostname), dbName(database), dbPort(port)
{
if(connectToDB() != 0)
THROWEXCEPTION("IpfixFlowInspectorExporter creation failed");
}
/**
* Destructor
*/
IpfixFlowInspectorExporter::~IpfixFlowInspectorExporter()
{
writeToDb();
}
#endif /* MONGO_SUPPORT_ENABLED */

90
src/modules/ipfix/database/IpfixFlowInspectorExporter.hpp Normal file
View File

@ -0,0 +1,90 @@
/*
* IPFIX Database Writer Redis Connector
* Copyright (C) 2012 Lothar Braun <braun@net.in.tum.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifdef REDIS_SUPPORT_ENABLED
#ifndef IPFIX_DB_WRITER_REDIS_H_
#define IPFIX_DB_WRITER_REDIS_H_
#include "IpfixDbCommon.hpp"
#include "../IpfixRecordDestination.h"
#include "common/ipfixlolib/ipfix.h"
#include "common/ipfixlolib/ipfixlolib.h"
#include <iostream>
#include <iomanip>
#include <stdlib.h>
#include <stdio.h>
#include <netinet/in.h>
#include <time.h>
#include <sstream>
#include <vector>
#include <hiredis/hiredis.h>
#define EXPORTERID 0
/**
* IpfixFlowInspectorExporter powered the communication to the redis database server
* also between the other structs
*/
class IpfixFlowInspectorExporter
: public IpfixRecordDestination, public Module, public Source<NullEmitable*>
{
public:
IpfixFlowInspectorExporter(const string& hostname, const string& database,
unsigned port);
~IpfixFlowInspectorExporter();
void onDataRecord(IpfixDataRecord* record);
/**
* Struct to identify the relationship between columns names and
* IPFIX_TYPEID, column type and default value
*/
struct Column {
const char* columnName; /** column name */
const char* columnType; /** column data type in database */
uint64_t defaultValue; /** default value */
InformationElement::IeId ipfixId; /** IPFIX_TYPEID */
InformationElement::IeEnterpriseNumber enterprise; /** enterprise number */
};
private:
// database data
std::string dbHost, dbName;
unsigned dbPort;
redisContext* context;
bool dbError;
std::list<std::string> bufferedObjects;
int connectToDB();
void processDataDataRecord(const IpfixRecord::SourceID& sourceID,
TemplateInfo& dataTemplateInfo, uint16_t length,
IpfixRecord::Data* data);
int writeToDb();
uint64_t getData(InformationElement::IeInfo type, IpfixRecord::Data* data);
std::string getInsertObj(TemplateInfo& dataTemplateInfo,uint16_t length, IpfixRecord::Data* data);
const static Column identify[];
};
#endif
#endif

78
src/modules/ipfix/database/IpfixFlowInspectorExporterCfg.cpp Normal file
View File

@ -0,0 +1,78 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2012 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
#ifdef REDIS_SUPPORT_ENABLED
#include "IpfixFlowInspectorExporterCfg.h"
#include <cassert>
IpfixFlowInspectorExporterCfg* IpfixFlowInspectorExporterCfg::create(XMLElement* e)
{
assert(e);
assert(e->getName() == getName());
return new IpfixFlowInspectorExporterCfg(e);
}
IpfixFlowInspectorExporterCfg::IpfixFlowInspectorExporterCfg(XMLElement* elem)
: CfgHelper<IpfixFlowInspectorExporter, IpfixFlowInspectorExporterCfg>(elem, "ipfixFlowInspectorExporter"),
port(6379)
{
if (!elem) return;
XMLNode::XMLSet<XMLElement*> set = _elem->getElementChildren();
for ( XMLNode::XMLSet<XMLElement*>::iterator it = set.begin(); it != set.end(); it++) {
XMLElement* e = *it;
if (e->matches("host")) {
hostname = e->getFirstText();
} else if (e->matches("port")) {
port = getInt("port");
} else if (e->matches("dbname")) {
database = e->getFirstText();
} else if (e->matches("next")) { // ignore next
} else {
msg(MSG_FATAL, "Unknown IpfixFlowInspectorExporter config statement %s\n", e->getName().c_str());
continue;
}
}
if (hostname=="") THROWEXCEPTION("IpfixFlowInspectorExporterCfg: host not set in configuration!");
if (database=="") THROWEXCEPTION("IpfixFlowInspectorExporterCfg: dbname not set in configuration!");
}
IpfixFlowInspectorExporterCfg::~IpfixFlowInspectorExporterCfg()
{
}
IpfixFlowInspectorExporter* IpfixFlowInspectorExporterCfg::createInstance()
{
instance = new IpfixFlowInspectorExporter(hostname, database, port);
msg(MSG_DEBUG, "IpfixFlowInspectorExporter configuration host %s queue %s port %i\n", hostname.c_str(), database.c_str(), port);
return instance;
}
bool IpfixFlowInspectorExporterCfg::deriveFrom(IpfixFlowInspectorExporterCfg* old)
{
return false;
}
#endif /*REDIS_SUPPORT_ENABLED*/

39
src/modules/ipfix/IpfixDbWriterPgCfg.h → src/modules/ipfix/database/IpfixFlowInspectorExporterCfg.h
View File

@ -1,6 +1,6 @@
/*
* Vermont Configuration Subsystem
* Copyright (C) 2009 Vermont Project
* Copyright (C) 2012 Vermont Project
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@ -18,47 +18,40 @@
*
*/
#ifndef IPFIXDBWRITERPGCFG_H_
#define IPFIXDBWRITERPGCFG_H_
#ifndef IPFIX_FLOW_INSPECTOR_EXPORTER_CFG_H_
#define IPFIX_FLOW_INSPECTOR_EXPORTER_CFG_H_
#ifdef PG_SUPPORT_ENABLED
#ifdef REDIS_SUPPORT_ENABLED
#include <core/XMLElement.h>
#include <core/Cfg.h>
#include "IpfixDbWriterPg.hpp"
#include "IpfixFlowInspectorExporter.hpp"
#include <string>
using namespace std;
class IpfixDbWriterPgCfg
: public CfgHelper<IpfixDbWriterPg, IpfixDbWriterPgCfg>
class IpfixFlowInspectorExporterCfg
: public CfgHelper<IpfixFlowInspectorExporter, IpfixFlowInspectorExporterCfg>
{
public:
friend class ConfigManager;
virtual IpfixDbWriterPgCfg* create(XMLElement* e);
virtual ~IpfixDbWriterPgCfg();
virtual IpfixFlowInspectorExporterCfg* create(XMLElement* e);
virtual ~IpfixFlowInspectorExporterCfg();
virtual IpfixDbWriterPg* createInstance();
virtual bool deriveFrom(IpfixDbWriterPgCfg* old);
virtual IpfixFlowInspectorExporter* createInstance();
virtual bool deriveFrom(IpfixFlowInspectorExporterCfg* old);
protected:
string hostname; /**< hostname of database host */
std::string hostname; /**< hostname of database host */
uint16_t port; /**< port of database */
string dbname; /**< database name */
string user; /**< user name for login to database */
string password; /**< password for login to database */
uint16_t bufferRecords; /**< amount of records to buffer until they are written to database */
std::string database; /**< mongo database name */
IpfixDbWriterPgCfg(XMLElement*);
IpfixFlowInspectorExporterCfg(XMLElement*);
};
#endif /*DB_SUPPORT_ENABLED*/
#endif /*REDIS_SUPPORT_ENABLED*/
#endif /*IPFIXDBWRITERCFG_H_*/
#endif /*IPFIX_FLOW_INSPECTOR_EXPORTER_CFG_H_*/

40
src/modules/packet/Observer.cpp
View File

@ -153,9 +153,7 @@ void *Observer::observerThread(void *arg)
struct pcap_pkthdr packetHeader;
bool have_send = false;
obs->registerCurrentThread();
bool file_eof = false;
bool file_eof = false;
msg(MSG_INFO, "Observer started with following parameters:");
msg(MSG_INFO, " - readFromFile=%d", obs->readFromFile);
@ -164,6 +162,7 @@ void *Observer::observerThread(void *arg)
msg(MSG_INFO, " - filterString='%s'", (obs->filter_exp ? obs->filter_exp : "none"));
msg(MSG_INFO, " - maxPackets=%u", obs->maxPackets);
msg(MSG_INFO, " - capturelen=%d", obs->capturelen);
msg(MSG_INFO, " - dataLinkType=%d", obs->dataLinkType);
if (obs->readFromFile) {
msg(MSG_INFO, " - autoExit=%d", obs->autoExit);
msg(MSG_INFO, " - stretchTime=%f", obs->stretchTime);
@ -216,7 +215,7 @@ void *Observer::observerThread(void *arg)
// initialize packet structure (init copies packet data)
p = packetManager.getNewInstance();
p->init((char*)pcapData, packetHeader.caplen, packetHeader.ts, obs->observationDomainID, packetHeader.len);
p->init((char*)pcapData, packetHeader.caplen, packetHeader.ts, obs->observationDomainID, packetHeader.len, obs->dataLinkType);
DPRINTF("received packet at %u.%04u, len=%d",
(unsigned)p->timestamp.tv_sec,
@ -312,7 +311,7 @@ void *Observer::observerThread(void *arg)
// in contrast to live capturing, the data length is not limited
// to any snap length when reading from a pcap file
(packetHeader.caplen < obs->capturelen) ? packetHeader.caplen : obs->capturelen,
packetHeader.ts, obs->observationDomainID, packetHeader.len);
packetHeader.ts, obs->observationDomainID, packetHeader.len, obs->dataLinkType);
DPRINTF("received packet at %u.%03u, len=%d",
@ -393,31 +392,6 @@ bool Observer::prepare(const std::string& filter)
goto out2;
}
// IP_HEADER_OFFSET is set by the configure script
switch (getDataLinkType()) {
case DLT_EN10MB:
if (IP_HEADER_OFFSET != 14 && IP_HEADER_OFFSET != 18) {
msg(MSG_FATAL, "IP_HEADER_OFFSET on an ethernet device has to be 14 or 18 Bytes. Please adjust that value via configure --with-ipheader-offset");
goto out2;
}
break;
case DLT_LOOP:
case DLT_NULL:
if (IP_HEADER_OFFSET != 4) {
msg(MSG_FATAL, "IP_HEADER_OFFSET on BSD loop back device has to be 4 Bytes. Please adjust that value via configure --with-ipheader-offset");
goto out2;
}
break;
case DLT_LINUX_SLL:
if (IP_HEADER_OFFSET != 16) {
msg(MSG_FATAL, "IP_HEADER_OFFSET on linux cooked devices has to be 16 Bytes. Please adjust that value via configure --with-ipheader-offset");
goto out2;
}
default:
msg(MSG_ERROR, "You are using an unkown IP_HEADER_OFFSET and data link combination. This can make problems. Please check if you use the correct IP_HEADER_OFFSET for your data link, if you see strange IPFIX/PSAMP packets.");
}
/* we need the netmask for the pcap_compile */
if(pcap_lookupnet(captureInterface, &network, &netmask, errorBuffer) == -1) {
msg(MSG_ERROR, "unable to determine netmask/network: %s", errorBuffer);
@ -439,6 +413,7 @@ bool Observer::prepare(const std::string& filter)
netmask=0;
}
dataLinkType = pcap_datalink(captureDevice);
if (filter_exp) {
msg(MSG_DEBUG, "compiling pcap filter code from: %s", filter_exp);
@ -533,11 +508,6 @@ int Observer::getCaptureLen()
return capturelen;
}
int Observer::getDataLinkType()
{
return pcap_datalink(captureDevice);
}
void Observer::replaceOfflineTimestamps()
{
replaceTimestampsFromFile = true;

3
src/modules/packet/Observer.h
View File

@ -58,7 +58,6 @@ public:
bool prepare(const std::string& filter);
static void doLogging(void *arg);
virtual std::string getStatisticsXML(double interval);
int getDataLinkType();
protected:
@ -132,6 +131,8 @@ protected:
uint32_t statTotalRecvPackets;
static void *observerThread(void *);
int dataLinkType; // contains the datalink type of the capturing device
};
#endif

114
src/modules/packet/Packet.h
View File

@ -24,6 +24,7 @@
#include <cstring>
#include <stdint.h>
#include <netinet/in.h>
#include <net/ethernet.h>
#include <sys/time.h>
#include "common/msg.h"
@ -32,6 +33,8 @@
#include "common/ManagedInstance.h"
#include "common/ipfixlolib/encoding.h"
#include <pcap.h>
#include "core/Emitable.h"
// the various header types (actually, HEAD_PAYLOAD is not neccessarily a header but it works like one for
@ -78,13 +81,6 @@
class Packet : public ManagedInstance<Packet>, public Emitable
{
public:
/*
the raw offset at which the IP header starts in the packet
for Ethernet, this is 14 bytes (MAC header size).
This constant is set via the configure script. It defaults to 14
*/
static const int IPHeaderOffset=IP_HEADER_OFFSET;
// Transport header classifications (used in Packet::ipProtocolType)
// Note: ALL is reserved and enables bitoperations using the enums
enum IPProtocolType { NONE=0x00, TCP=0x01, UDP=0x02, ICMP=0x04, IGMP=0x08, ALL=0xFF };
@ -138,54 +134,23 @@ public:
Packet(InstanceManager<Packet>* im)
: ManagedInstance<Packet>(im),
zeroBytes(0),
netHeader(data + IPHeaderOffset), // netHeader must not be changed afterwards
netHeaderOffset(IPHeaderOffset)
netHeader(data),
netHeaderOffset(0)
{
}
Packet()
: ManagedInstance<Packet>(0),
netHeader(data + IPHeaderOffset),
netHeaderOffset(IPHeaderOffset)
zeroBytes(0),
netHeader(data),
netHeaderOffset(0)
{
}
/*
void copyPacket(Packet* other)
{
observationDomainID = other->observationDomainID;
memcpy(&data, &other->data, PCAP_MAX_CAPTURE_LENGTH);
netHeader = other->netHeader;
transportHeader = other->transportHeader;
payload = other->payload;
zeroBytes = other->zeroBytes;
netHeaderOffset = other->netHeaderOffset;
transportHeaderOffset = other->transportHeaderOffset;
payloadOffset = other->payloadOffset;
classification = other->classification;
ipProtocolType = other->ipProtocolType;
data_length = other->data_length;
pcapPacketLength = other->pcapPacketLength;
timestamp = other->timestamp;
time_sec_nbo = other->time_sec_nbo;
time_usec_nbo = other->time_usec_nbo;
time_msec_nbo = other->time_msec_nbo;
memcpy(&varlength, &other->varlength, sizeof(varlength));
varlength_index = other->varlength_index;
}
*/
/**
* @param origplen original packet length
*/
inline void init(char* packetData, int len, struct timeval time, uint32_t obsdomainid, uint32_t origplen)
inline void init(char* packetData, int len, struct timeval time, uint32_t obsdomainid, uint32_t origplen, int dataLinkType)
{
transportHeader = NULL;
payload = NULL;
@ -199,12 +164,14 @@ public:
observationDomainID = obsdomainid;
pcapPacketLength = origplen;
if (len > PCAP_MAX_CAPTURE_LENGTH) {
THROWEXCEPTION("received packet of size %d is bigger than maximum length (%d), "
"adjust compile-time parameter PCAP_MAX_CAPTURE_LENGTH to compensate!", len, PCAP_MAX_CAPTURE_LENGTH);
static int offset = getLayer2HeaderLen(packetData, dataLinkType);
if (len > PCAP_MAX_CAPTURE_LENGTH || len < offset) {
THROWEXCEPTION("received packet of size %d is bigger than maximum length (%d) or smaller than layer 2 len (%d), "
"adjust compile-time parameter PCAP_MAX_CAPTURE_LENGTH to compensate!", len, PCAP_MAX_CAPTURE_LENGTH, offset);
}
memcpy(data, packetData, len);
// copy all content starting from the IP header
memcpy(data , packetData + offset, len - offset);
// timestamps in network byte order (needed for export or concentrator)
time_sec_nbo = htonl(timestamp.tv_sec);
@ -217,10 +184,10 @@ public:
totalPacketsReceived++;
classify();
classify(dataLinkType);
};
inline void init(char** datasegments, uint32_t* segmentlens, struct timeval time, uint32_t obsdomainid, uint32_t origplen)
inline void init(char** datasegments, uint32_t* segmentlens, struct timeval time, uint32_t obsdomainid, uint32_t origplen, int dataLinkType)
{
transportHeader = NULL;
payload = NULL;
@ -233,16 +200,22 @@ public:
observationDomainID = obsdomainid;
pcapPacketLength = origplen;
data_length = 0;
static int offset = getLayer2HeaderLen(datasegments[0], dataLinkType);
for (uint32_t i=0; datasegments[i]!=0; i++) {
if (data_length+segmentlens[i] > PCAP_MAX_CAPTURE_LENGTH) {
THROWEXCEPTION("received packet of size %d is bigger than maximum length (%d), "
"adjust compile-time parameter PCAP_MAX_CAPTURE_LENGTH to compensate!", data_length+segmentlens[i], PCAP_MAX_CAPTURE_LENGTH);
}
memcpy(data+data_length, datasegments[i], segmentlens[i]);
data_length += segmentlens[i];
if (i == 0) {
// first segment contains the layer 2 header
// we want to skip it
memcpy(data+data_length, datasegments[i] + offset, segmentlens[i] - offset);
data_length += segmentlens[i] - offset;
} else {
memcpy(data+data_length, datasegments[i], segmentlens[i]);
data_length += segmentlens[i];
}
}
// timestamps in network byte order (needed for export or concentrator)
@ -256,7 +229,7 @@ public:
totalPacketsReceived++;
classify();
classify(dataLinkType);
};
// Delete the packet and free all data associated with it.
@ -264,8 +237,35 @@ public:
{
}
int getLayer2HeaderLen(const char* packetData, int dataLinkType)
{
// get netheader
switch (dataLinkType) {
case DLT_EN10MB: {
// 14 oder 18
// check if we have a vlan on the data link layer.
uint16_t et = ntohs(((struct ether_header*)packetData)->ether_type);
return 14 + ((et == ETHERTYPE_VLAN)?4:0);
break;
}
case DLT_LOOP:
case DLT_NULL: {
return 4;
break;
}
case DLT_LINUX_SLL:{
return 16;
break;
}
default:
msg(MSG_ERROR, "Received packet on not supported link layer \"%u\". Assuming that no layer 2 header exists. This will probably fail!", dataLinkType);
return 0;
}
return 0;
}
// classify the packet headers
void classify()
void classify(int dataLinkType)
{
unsigned char protocol = 0;

13
src/osdep/freebsd/iphdrs.h
View File

@ -25,6 +25,11 @@
#define _FREEBSD_IPHDR_
#if defined(__FreeBSD__) || defined(__APPLE__)
#if defined(iphdr)
#undef iphdr
#endif
struct iphdr
{
#if __BYTE_ORDER == __LITTLE_ENDIAN
@ -48,6 +53,10 @@ struct iphdr
/*The options start here. */
};
#ifdef udphdr
#undef udphdr
#endif
struct udphdr
{
u_int16_t source;
@ -56,6 +65,10 @@ struct udphdr
u_int16_t check;
};
#ifdef tcphdr
#undef tcphdr
#endif
struct tcphdr
{
u_int16_t source;

2
src/tests/vermonttest/AggregationPerfTest.cpp
View File

@ -146,7 +146,7 @@ void AggregationPerfTest::sendPacketsTo(Destination<Packet*>* dest, uint32_t num
for (size_t i = 0; i < numpackets; i++) {
Packet* packet = packetManager.getNewInstance();
packet->init((char*)packetdata, packetdatalen, curtime, 0, packetdatalen);
packet->init((char*)packetdata, packetdatalen, curtime, 0, packetdatalen, 14);
dest->receive(packet);
}
}

2
src/tests/vermonttest/ReconfTest.cpp
View File

@ -43,7 +43,7 @@ void ReconfTest::sendPacketsTo(Destination<Packet*>* dest, size_t numpackets)
for (size_t i = 0; i < numpackets; i++) {
Packet* packet = packetManager.getNewInstance();
packet->init((char*)packetdata, packetdatalen, curtime, 0, packetdatalen);
packet->init((char*)packetdata, packetdatalen, curtime, 0, packetdatalen, 14);
dest->receive(packet);
}
}

7
tools/injectUDPToCollector.cpp
View File

@ -34,8 +34,15 @@
#include <string>
#include <iostream>
#include <pcap.h>
#include <netinet/ip.h>
#if defined(__FreeBSD__) || defined(__APPLE__)
#include <osdep/freebsd/iphdrs.h>
#else
#include <netinet/udp.h>
#endif
#include <net/if.h>
#include <netinet/if_ether.h>
#include <sys/socket.h>