Menu
▾
▴
[aperture-commit] SF.net SVN: aperture:[2222] aperture-webserver/trunk
|
From: <leo...@us...> - 2010-02-16 22:26:05
|
Revision: 2222
http://aperture.svn.sourceforge.net/aperture/?rev=2222&view=rev
Author: leo_sauermann
Date: 2010-02-16 22:25:57 +0000 (Tue, 16 Feb 2010)
Log Message:
-----------
aperture-webserver: fixed logging of crawling messages, added more reporting, added statistics page, added page listing all crawled resources+dates+bytesize
Modified Paths:
--------------
aperture-webserver/trunk/.settings/org.eclipse.wst.common.component
aperture-webserver/trunk/WebContent/config/index.jsp
aperture-webserver/trunk/WebContent/config/sourcedetails.jsp
aperture-webserver/trunk/WebContent/config/style.css
aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalCrawlerHandler.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalXmlRpcService.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/ApertureServer.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/DataSourceLog.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/bean/DataSourceUIBean.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/dao/DataSourceInstanceDAO.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourceInformation.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourcePool.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/exception/DataSourceNotFoundException.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/impl/ServerImpl.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/thread/CrawlingThread.java
aperture-webserver/trunk/src/org/semanticdesktop/aperture/servlet/ApertureServerServlet.java
aperture-webserver/trunk/src.test/org/semanticdesktop/aperture/server/DataSourceLogTest.java
Added Paths:
-----------
aperture-webserver/trunk/WebContent/config/datasourceaccess.jsp
aperture-webserver/trunk/WebContent/config/datasourcereport.jsp
Removed Paths:
-------------
aperture-webserver/trunk/WebContent/config/sourcestatistics.jsp
Modified: aperture-webserver/trunk/.settings/org.eclipse.wst.common.component
===================================================================
--- aperture-webserver/trunk/.settings/org.eclipse.wst.common.component 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/.settings/org.eclipse.wst.common.component 2010-02-16 22:25:57 UTC (rev 2222)
@@ -3,6 +3,7 @@
<wb-module deploy-name="aperture-webserver">
<wb-resource deploy-path="/" source-path="/WebContent"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src"/>
+ <wb-resource deploy-path="/WEB-INF/classes" source-path="/src.test"/>
<property name="java-output-path" value="build/classes"/>
<property name="context-root" value="aperture-webserver"/>
</wb-module>
Added: aperture-webserver/trunk/WebContent/config/datasourceaccess.jsp
===================================================================
--- aperture-webserver/trunk/WebContent/config/datasourceaccess.jsp (rev 0)
+++ aperture-webserver/trunk/WebContent/config/datasourceaccess.jsp 2010-02-16 22:25:57 UTC (rev 2222)
@@ -0,0 +1,99 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+<%@ taglib prefix="aperture" uri="http://nepomuk.semanticdesktop.org/datawrapper/aperture/taglib"%>
+<%@page import="java.net.URLDecoder"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.bean.FresnelEditor"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceDAOFactory"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.bean.DataSourceUIBean"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceInstanceDAO"%>
+<%@page import="org.semanticdesktop.aperture.servlet.ApertureServerServlet"%>
+<%@page import="org.semanticdesktop.aperture.server.ApertureServer"%>
+<%@page import="org.semanticdesktop.aperture.server.CrawlerStateBean"%>
+<%@page import="org.semanticdesktop.aperture.server.DataSourceLog"%>
+<%@page import="java.util.logging.Level"%>
+<%@page import="java.util.Collection"%>
+<%@page import="java.util.logging.LogRecord"%>
+<%@page import="java.util.Date"%>
+<%@page import="java.text.DateFormat"%>
+<%@page import="java.text.SimpleDateFormat"%>
+<%
+/**
+ * render the dataaccess statistics of a datasource
+ * @author leo sauermann
+ */
+
+// Authenticate
+if (!ApertureServerServlet.authenticateAdministrator(request, response, getServletContext()))
+ return;
+
+// Get URI of datasource
+String uriString = (String)request.getParameter("uri");
+if (uriString == null) {
+ throw new ServletException("parameter 'uri' identifying datasource missing");
+}
+String uriDecoded = URLDecoder.decode(uriString,"UTF-8");
+
+ApertureServer server = ApertureServerServlet.getApertureServer(getServletContext());
+
+SimpleDateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss");
+
+DataSourceUIBean datasource = server.getDataSourceBean(uriDecoded);
+if (datasource != null)
+ request.setAttribute("datasource", datasource);
+
+%>
+
+<%@page import="java.util.Set"%>
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+ <title>Reports for ${datasource.name}</title>
+ <link title="default" rel="stylesheet" type="text/css"
+ href="style.css" media="screen" />
+</head>
+<body>
+<% if (datasource == null) { %>
+ <center>
+ <h1>Data source not found</h1>
+ <h2><a href="index.jsp">Back to the sources list</a></h2>
+ </center>
+<% } else { %>
+ <h1>All resources of <a href="sourcedetails.jsp?uri=${datasource.dataSourceURI}">'${datasource.name}' data source</a></h1>
+ <table>
+ <tr><td>URI</td><td>last access</td><td>size</td></tr>
+ <%
+ Set ids = datasource.getAccessDataIDSet();
+ for (Object id : ids) {
+ String idv = id.toString();
+ request.setAttribute("accid", idv);
+ try {
+ request.setAttribute("accsize", datasource.getAccessDataByteSize(idv));
+ } catch (Exception x) {
+ // swallow
+ }
+ try {
+ long millis = datasource.getAccessDataLastAccess(idv);
+ if (millis != 0)
+ {
+ Date d = new Date(millis);
+ request.setAttribute("accdate", dateformat.format(d));
+ } else
+ request.setAttribute("accdate", "0");
+ } catch (Exception x) {
+ // swallow
+ }
+
+
+ %>
+ <tr><td>${accid}</td><td>${accdate}</td><td>${accsize}</td></tr>
+ <%
+ }
+ %>
+ </table>
+ </p>
+
+
+ <h2><a href="index.jsp">Back to the data sources list</a></h2>
+<% datasource.dispose();
+ } %>
+</body>
+</html>
Copied: aperture-webserver/trunk/WebContent/config/datasourcereport.jsp (from rev 2221, aperture-webserver/trunk/WebContent/config/sourcestatistics.jsp)
===================================================================
--- aperture-webserver/trunk/WebContent/config/datasourcereport.jsp (rev 0)
+++ aperture-webserver/trunk/WebContent/config/datasourcereport.jsp 2010-02-16 22:25:57 UTC (rev 2222)
@@ -0,0 +1,140 @@
+<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
+<%@ taglib prefix="aperture" uri="http://nepomuk.semanticdesktop.org/datawrapper/aperture/taglib"%>
+<%@page import="java.net.URLDecoder"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.bean.FresnelEditor"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceDAOFactory"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.bean.DataSourceUIBean"%>
+<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceInstanceDAO"%>
+<%@page import="org.semanticdesktop.aperture.servlet.ApertureServerServlet"%>
+<%@page import="org.semanticdesktop.aperture.server.ApertureServer"%>
+<%@page import="org.semanticdesktop.aperture.server.CrawlerStateBean"%>
+<%@page import="org.semanticdesktop.aperture.server.DataSourceLog"%>
+<%@page import="java.util.logging.Level"%>
+<%
+/**
+ * render the detail statistics of a datasource
+ * @author leo sauermann
+ */
+
+// Authenticate
+if (!ApertureServerServlet.authenticateAdministrator(request, response, getServletContext()))
+ return;
+
+// Get URI of datasource
+String uriString = (String)request.getParameter("uri");
+if (uriString == null) {
+ throw new ServletException("parameter 'uri' identifying datasource missing");
+}
+String uriDecoded = URLDecoder.decode(uriString,"UTF-8");
+
+ApertureServer server = ApertureServerServlet.getApertureServer(getServletContext());
+
+SimpleDateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss");
+
+DataSourceUIBean datasource = server.getDataSourceBean(uriDecoded);
+if (datasource != null)
+ request.setAttribute("datasource", datasource);
+CrawlerStateBean crawlerstate = server.getDetailedCrawlerState(uriDecoded);
+if (crawlerstate != null)
+ request.setAttribute("crawlerstate", crawlerstate);
+DataSourceLog dslog = server.getDataSourceLog(uriDecoded);
+if (dslog != null)
+ request.setAttribute("dslog", crawlerstate);
+
+%>
+
+<%@page import="java.util.Collection"%>
+<%@page import="java.util.logging.LogRecord"%>
+<%@page import="java.util.Date"%>
+<%@page import="java.text.DateFormat"%>
+<%@page import="java.text.SimpleDateFormat"%>
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
+ <title>Reports for ${datasource.name}</title>
+ <link title="default" rel="stylesheet" type="text/css"
+ href="style.css" media="screen" />
+</head>
+<body>
+<% if (datasource == null) { %>
+ <center>
+ <h1>Data source not found</h1>
+ <h2><a href="index.jsp">Back to the sources list</a></h2>
+ </center>
+<% } else { %>
+ <h1>Report of <a href="sourcedetails.jsp?uri=${datasource.dataSourceURI}">'${datasource.name}' data source</a></h1>
+ <ul>
+ <li><a href="#lastreport">Last crawl report</a></li>
+ <li><a href="#statistics">Statistics</a></li>
+ <li><a href="#logmessages">Log messages</a></li>
+ </ul>
+ <%if (datasource.getLastErrorMessage() != null) {%>
+ <h2>Error message</h2>
+ <table class="sourcedetails">
+ <TR>
+ <TD class="errormsg">
+ <pre>${bean.lastErrorMessage}</pre>
+ </TD>
+ </TR>
+ </table>
+ <%} %>
+
+ <h2 id="lastreport">Last Crawl report</h2>
+ <table>
+ <TR>
+ <TD class="rowheader">Crawl started:</TD>
+ <td>${datasource.crawlStartedString}</td>
+ </TR>
+ <TR>
+ <TD class="rowheader">Crawl stopped:</TD>
+ <td>${datasource.crawlStoppedString}</td>
+ </TR>
+ <TR>
+ <TD class="rowheader">New objects:</TD>
+ <td>${datasource.newObjects}</td>
+ </TR>
+ <TR>
+ <TD class="rowheader">Changed objects:</TD>
+ <td>${datasource.modifiedObjects}</td>
+ </TR>
+ <TR>
+ <TD class="rowheader">Unchanged objects:</TD>
+ <td>${datasource.unmodifiedObjects}</td>
+ </TR>
+ <TR>
+ <TD class="rowheader">Removed objects:</TD>
+ <td>${datasource.deletedObjects}</td>
+ </TR>
+ </table>
+
+ <h2 id="statistics">Statistics</h2>
+ <table>
+ <tr><td class="rowheader">resources</td><td>${datasource.accessDataSize} resources were crawled</td></tr>
+ <tr><td> </td><td><a href="datasourceaccess.jsp?uri=${datasource.dataSourceURI}">list all</a> <b>Attention</b>: may be huge</td></tr>
+ </table>
+
+
+ <h2 id="logmessages">Log Messages</h2>
+ <p>
+ <table width="100%" border="0">
+ <tr><td>Level</td><td>Date</td><td>Message</td></tr>
+ <%
+ Collection<LogRecord> logs = dslog.read(100, Level.INFO);
+ for (LogRecord log : logs) {
+ request.setAttribute("log", log);
+ Date logdate = new Date(log.getMillis());
+ request.setAttribute("logdate", dateformat.format(logdate));
+ %>
+ <tr><td>${log.level}</td><td>${logdate}</td><td>${log.message}</td></tr>
+ <%
+ }
+ %>
+ </table>
+ </p>
+
+
+ <h2><a href="index.jsp">Back to the data sources list</a></h2>
+<% datasource.dispose();
+ } %>
+</body>
+</html>
Property changes on: aperture-webserver/trunk/WebContent/config/datasourcereport.jsp
___________________________________________________________________
Added: svn:mergeinfo
+
Modified: aperture-webserver/trunk/WebContent/config/index.jsp
===================================================================
--- aperture-webserver/trunk/WebContent/config/index.jsp 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/WebContent/config/index.jsp 2010-02-16 22:25:57 UTC (rev 2222)
@@ -49,12 +49,12 @@
<h2>Currenly configured data sources.</h2>
<table>
<tr class="tableheader">
- <td>Name:</td>
- <td>Type:</td>
- <td>Status:</td>
- <td>Last crawled:</td>
- <td>Next Crawl</td>
- <td colspan="3">Actions:</td>
+ <td>Name</td>
+ <td>Type</td>
+ <td>Status</td>
+ <td>Last crawled</td>
+ <td>Next crawl</td>
+ <td colspan="3">Actions</td>
</tr>
<c:forEach items="${sourcesList}" var="source">
<tr>
Modified: aperture-webserver/trunk/WebContent/config/sourcedetails.jsp
===================================================================
--- aperture-webserver/trunk/WebContent/config/sourcedetails.jsp 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/WebContent/config/sourcedetails.jsp 2010-02-16 22:25:57 UTC (rev 2222)
@@ -70,7 +70,9 @@
</table>
<%} %>
- <h2>Crawl report</h2>
+ <h2>Reports</h2>
+ <a href="datasourcereport.jsp?uri=${bean.dataSourceURI}"><b>detailed report<b></a>
+ <h3>Crawl report</h3>
<table >
<TR>
<TD class="rowheader">Crawl started:</TD>
Deleted: aperture-webserver/trunk/WebContent/config/sourcestatistics.jsp
===================================================================
--- aperture-webserver/trunk/WebContent/config/sourcestatistics.jsp 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/WebContent/config/sourcestatistics.jsp 2010-02-16 22:25:57 UTC (rev 2222)
@@ -1,111 +0,0 @@
-<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%>
-<%@ taglib prefix="aperture" uri="http://nepomuk.semanticdesktop.org/datawrapper/aperture/taglib"%>
-<%@page import="java.net.URLDecoder"%>
-<%@page import="org.semanticdesktop.aperture.server.configui.bean.FresnelEditor"%>
-<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceDAOFactory"%>
-<%@page import="org.semanticdesktop.aperture.server.configui.bean.DataSourceUIBean"%>
-<%@page import="org.semanticdesktop.aperture.server.configui.dao.DataSourceInstanceDAO"%>
-<%@page import="org.semanticdesktop.aperture.servlet.ApertureServerServlet"%>
-<%@page import="org.semanticdesktop.aperture.server.ApertureServer"%>
-<%@page import="org.semanticdesktop.aperture.server.CrawlerStateBean"%>
-<%@page import="org.semanticdesktop.aperture.server.DataSourceLog"%>
-
-<%
-/**
- * render the detail statistics of a datasource
- * @author leo sauermann
- */
-
-// Authenticate
-if (!ApertureServerServlet.authenticateAdministrator(request, response, getServletContext()))
- return;
-
-// Get URI of datasource
-String uriString = (String)request.getParameter("uri");
-if (uriString == null) {
- throw new ServletException("parameter 'uri' identifying datasource missing");
-}
-String uriDecoded = URLDecoder.decode(uriString,"UTF-8");
-
-ApertureServer server = ApertureServerServlet.getApertureServer(getServletContext());
-CrawlerStateBean crawlerstate = server.getDetailedCrawlerState(uriDecoded);
-if (crawlerstate != null)
- request.setAttribute("crawlerstate", crawlerstate);
-DataSourceLog dslog = server.getDataSourceLog(uriDecoded);
-%>
-
-
-<html>
-<head>
- <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
- <title>Details for ${bean.name}</title>
- <link title="default" rel="stylesheet" type="text/css"
- href="style.css" media="screen" />
-</head>
-<body>
-<% if (bean == null) { %>
- <center>
- <h1>Data source not found</h1>
- <h2><a href="index.jsp">Back to the sources list</a></h2>
- </center>
-<% } else { %>
- <h1>Details of '${bean.name}' data source</h1>
-
- <h2>Basic data source information</h2>
- <%= fresnelEditor.createCommonFormPart() %>
- <TR>
- <TD class="rowheader">Status:</TD>
- <aperture:statuscell state="${bean.crawlingState}"/>
- </TR>
- </table>
-
- <h2>Detailed data source configuration</h2>
-
- <%= fresnelEditor.createSpecificFormPart() %>
-
- <h2><aperture:crawlbutton bean="${bean}"/></h2>
-
- <%if (bean.getLastErrorMessage() != null) {%>
- <h2>Error message</h2>
- <table class="sourcedetails">
- <TR>
- <TD class="errormsg">
- <pre>${bean.lastErrorMessage}</pre>
- </TD>
- </TR>
- </table>
- <%} %>
-
- <h2>Crawl report</h2>
- <table >
- <TR>
- <TD class="rowheader">Crawl started:</TD>
- <td class="configinput">${bean.crawlStartedString}</td>
- </TR>
- <TR>
- <TD class="rowheader">Crawl stopped:</TD>
- <td class="configinput">${bean.crawlStoppedString}</td>
- </TR>
- <TR>
- <TD class="rowheader">New objects:</TD>
- <td class="configinput">${bean.newObjects}</td>
- </TR>
- <TR>
- <TD class="rowheader">Changed objects:</TD>
- <td class="configinput">${bean.modifiedObjects}</td>
- </TR>
- <TR>
- <TD class="rowheader">Unchanged objects:</TD>
- <td class="configinput">${bean.unmodifiedObjects}</td>
- </TR>
- <TR>
- <TD class="rowheader">Removed objects:</TD>
- <td class="configinput">${bean.deletedObjects}</td>
- </TR>
- </table>
-
- <h2><a href="index.jsp">Back to the data sources list</a></h2>
-<% bean.dispose();
- } %>
-</body>
-</html>
Modified: aperture-webserver/trunk/WebContent/config/style.css
===================================================================
--- aperture-webserver/trunk/WebContent/config/style.css 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/WebContent/config/style.css 2010-02-16 22:25:57 UTC (rev 2222)
@@ -7,20 +7,20 @@
}
body {
- margin-left: 50px;
- margin-right: 50px;
- padding: 50px 38px 0 37px;
+ margin-left: 10px;
+ margin-right: 10px;
+ padding: 0px 0px 0px 0px;
}
h1 {
- text-align: center;
+ text-align: left;
font-size: 200%;
}
h2 {
margin: 20px 0 15px 0;
padding: 0;
- text-align: center;
+ text-align: left;
font-size: 130%;
}
@@ -37,10 +37,6 @@
margin-right:auto;
}
-table {
- margin-left: auto;
- margin-right: auto;
-}
table.sourcedetails {
width: 590px
@@ -53,7 +49,6 @@
tr.tableheader {
font-weight:bold;
- text-align:center;
}
td.rowheader {
@@ -100,17 +95,3 @@
vertical-align: top;
}
-a:link {
- text-decoration: none;
- color: #CC0000;
-}
-
-a:visited {
- text-decoration: none;
- color: #CC6666;
-}
-
-a:hover {
- text-decoration: underline;
- color: #CC0000;
-}
\ No newline at end of file
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalCrawlerHandler.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalCrawlerHandler.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalCrawlerHandler.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -32,12 +32,12 @@
* Testing parameters
* @param args
*/
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
DrupalXmlRpcService service = new DrupalXmlRpcService(
"localhost",
"dac5b06a2e63eed9336ce24f5e56a181",
"http://localhost/organikdrupal/?q=services/xmlrpc");
- if(service.connect() == true)
+ service.connect();
{
service.login("root", "root");
try {
@@ -134,15 +134,18 @@
public void connect() throws Exception {
service = new DrupalXmlRpcService(domain, apikey, serviceUrl);
- if(!service.connect())
- throw new Exception("Cannot connect to '"+serviceUrl+"'");
+ service.connect();
service.login(username, password);
}
public void dispose() {
if (service != null) {
- service.logout();
+ try {
+ service.logout();
+ } catch (Exception e) {
+ // swallow
+ }
service = null;
}
}
@@ -193,8 +196,14 @@
RDFContainer data = object.getMetadata();
DrupalNode node = new DrupalNode();
node.setType(DrupalNode.TYPE_STORY);
- node.setTitle(data.getString(NIE.title));
- node.setBody(data.getString(NIE.plainTextContent));
+ String title = data.getString(NIE.title);
+ if (title == null)
+ title = object.getID().toString();
+ node.setTitle(title);
+ String body = data.getString(NIE.plainTextContent);
+ if (body == null)
+ body = " ";
+ node.setBody(body);
service.nodeSave(node);
} catch (Exception x) {
logger.warn("cannot store extracted content from '"+object.getID()+"' to drupal: "+x,x);
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalXmlRpcService.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalXmlRpcService.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/drupalhandler/DrupalXmlRpcService.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -1,8 +1,5 @@
package org.semanticdesktop.aperture.drupalhandler;
-
-
-
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.logging.Level;
@@ -18,257 +15,242 @@
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
/**
-* see http://drupal.org/node/632844
-* @author Aaron Moline <Aar...@mo...>
-*/
+ * DrupalXmlRpcService
+ *
+ * see http://drupal.org/node/632844, with adaptations by Leo Sauermann
+ *
+ * Changelog 16.2.2010 - made exceptions where exceptions are due, replaced bad
+ * logging with good logging JUL
+ *
+ * @author Aaron Moline <Aar...@mo...>
+ * @author Leo Sauermann <leo...@df...>
+ */
public class DrupalXmlRpcService {
- // <editor-fold desc="Public Properties">
- public String ServiceURL;
- public String ServiceDomain;
- public String ApiKey;
- public String Nonce;
- public long TimeStamp;
- public String APIHash;
- public XmlRpcClient XmlService;
- public boolean APIKey_Active;
- // </editor-fold>
+ Logger log = Logger.getLogger(DrupalXmlRpcService.class.getName());
- // <editor-fold desc=" Constructors ">
- public DrupalXmlRpcService(String serviceDomain, String apiKey, String serviceURL,boolean api_active) {
- this(serviceDomain, apiKey, serviceURL);
- this.APIKey_Active = api_active;
+ public String ServiceURL;
+ public String ServiceDomain;
+ public String ApiKey;
+ public String Nonce;
+ public long TimeStamp;
+ public String APIHash;
+ public XmlRpcClient XmlService;
+ public boolean APIKey_Active;
- }
+ public DrupalXmlRpcService(String serviceDomain, String apiKey,
+ String serviceURL, boolean api_active) {
+ this(serviceDomain, apiKey, serviceURL);
+ this.APIKey_Active = api_active;
- public DrupalXmlRpcService(String serviceDomain, String apiKey, String serviceURL) {
- this.ServiceDomain = serviceDomain;
- this.ApiKey = apiKey;
- this.ServiceURL = serviceURL;
- this.APIKey_Active = true;
- }
- // </editor-fold>
+ }
- // <editor-fold defaultstate="collapsed" desc="Private Properties">
- private String SessionID;
- // </editor-fold>
+ public DrupalXmlRpcService(String serviceDomain, String apiKey,
+ String serviceURL) {
+ this.ServiceDomain = serviceDomain;
+ this.ApiKey = apiKey;
+ this.ServiceURL = serviceURL;
+ this.APIKey_Active = true;
+ }
- // <editor-fold defaultstate="collapsed" desc="Private Methods">
- private String GetNonce()/*(int length)*/ {
- /*
- * //TODO:Get None Generator Working
- String allowedCharacters = "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789";
- StringBuilder password = new StringBuilder();
+ private String SessionID;
- Random rand = new Random();
- for (int i = 0; i < length; i++)
- {
- password.append()
- //password.append(append);
- }
+ private String getNonce()/* (int length) */{
+ /*
+ * //TODO:Get None Generator Working String allowedCharacters =
+ * "abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+ * StringBuilder password = new StringBuilder();
+ *
+ * Random rand = new Random(); for (int i = 0; i < length; i++) {
+ * password.append() //password.append(append); }
+ *
+ * return password.toString();
+ */
+ return "" + System.currentTimeMillis();
+ }
- return password.toString();
- * */
- return ""+System.currentTimeMillis();
- }
- /*
- *
- * @drupalServiceCommand
- *
- */
- private void intializeHash(DrupalServiceCommands drupalServiceCommand) {
- this.Nonce = GetNonce();
- this.TimeStamp = System.currentTimeMillis();
- String hashstring = GetHashString(drupalServiceCommand.toString());
- this.APIHash = GetHMAC(hashstring);
- }
+ /*
+ *
+ * @drupalServiceCommand
+ *
+ */
+ private void intializeHash(DrupalServiceCommands drupalServiceCommand)
+ throws Exception {
+ this.Nonce = getNonce();
+ this.TimeStamp = System.currentTimeMillis();
+ String hashstring = getHashString(drupalServiceCommand.toString());
+ this.APIHash = getHMAC(hashstring);
+ }
- private void IntializeService() {
- try {
+ private void intializeService() throws Exception {
+ XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl();
+ config.setServerURL(new URL(this.ServiceURL));
+ XmlService = new XmlRpcClient();
+ XmlService.setConfig(config);
+ }
- XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl();
- config.setServerURL(new URL(this.ServiceURL));
- XmlService = new XmlRpcClient();
- XmlService.setConfig(config);
+ private String getHashString(String serviccmd) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("" + System.currentTimeMillis()); // Time stamp
+ sb.append(";");
+ sb.append(this.ServiceDomain);// Service Domain
+ sb.append(";");
+ sb.append("" + System.currentTimeMillis()); // Nonce
+ sb.append(";");
+ sb.append(serviccmd); // Service command
+ log.finest("Created GetHashString: " + sb.toString());
+ return sb.toString();
+ }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ private Vector getDefaultParams() {
+ Vector params = new Vector();
+ // Drupal is setup to use Service Keys, then add the following.
+ if (this.APIKey_Active) {
+ params.add(this.APIHash);
+ params.add(this.ServiceDomain);
+ params.add("" + this.TimeStamp);
+ params.add(this.Nonce);
+ params.add(this.SessionID);
+ }
+ return params;
+ }
- private String GetHashString(String serviccmd) {
- StringBuilder sb = new StringBuilder();
- sb.append(""+System.currentTimeMillis()); // Time stamp
- sb.append(";");
- sb.append(this.ServiceDomain);//Service Domain
- sb.append(";");
- sb.append(""+System.currentTimeMillis()); //Nonce
- sb.append(";");
- sb.append(serviccmd); //Service command
- Debug.println("Created GetHashString: ",sb.toString());
- return sb.toString();
- }
+ // </editor-fold>
- private Vector GetDefaultParams() {
- Vector params = new Vector();
- //Drupal is setup to use Service Keys, then add the following.
- if(this.APIKey_Active)
- {
- params.add(this.APIHash);
- params.add(this.ServiceDomain);
- params.add(""+this.TimeStamp);
- params.add(this.Nonce);
- params.add(this.SessionID);
- }
- return params;
- }
- // </editor-fold>
+ // <editor-fold defaultstate="collapsed" desc="Public Methods ">
+ public String getHMAC(String message) throws Exception {
- // <editor-fold defaultstate="collapsed" desc="Public Methods ">
- public String GetHMAC(String message) {
+ Mac mac;
+ try {
+ Charset csets = Charset.forName("US-ASCII");
+ SecretKeySpec keySpec = new javax.crypto.spec.SecretKeySpec(csets
+ .encode(this.ApiKey).array(), "HmacSHA256");
+ mac = javax.crypto.Mac.getInstance("HmacsSHA256");
+ mac.init(keySpec);
+ byte[] hash = mac.doFinal(csets.encode(message).array());
- Mac mac;
- try {
- Charset csets = Charset.forName("US-ASCII");
- SecretKeySpec keySpec = new javax.crypto.spec.SecretKeySpec(csets.encode(this.ApiKey).array(), "HmacSHA256");
- mac = javax.crypto.Mac.getInstance("HmacSHA256");
- mac.init(keySpec);
- byte[] hash = mac.doFinal(csets.encode(message).array());
+ String result = "";
+ for (int i = 0; i < hash.length; i++) {
+ result += Integer.toString((hash[i] & 0xff) + 0x100, 16)
+ .substring(1);
+ }
- String result = "";
- for (int i = 0; i < hash.length; i++) {
- result += Integer.toString((hash[i] & 0xff) + 0x100, 16).substring(1);
- }
+ log.finest("Created HMAC: " + result);
+ return result;
- Debug.println("Created HMAC: ", result);
- return result;
+ } catch (Exception ex) {
+ log.log(Level.SEVERE, ex.getLocalizedMessage(), ex);
+ throw ex;
+ }
- } catch (InvalidKeyException ex) {
- Logger.getLogger(DrupalXmlRpcService.class.getName()).log(Level.SEVERE, null, ex);
- throw new RuntimeException(ex);
- } catch (NoSuchAlgorithmException ex) {
- Logger.getLogger(DrupalXmlRpcService.class.getName()).log(Level.SEVERE, null, ex);
- throw new RuntimeException(ex);
- }
+ }
+ /**
+ * Connect to the remote service
+ *
+ * @return
+ * @throws Exception
+ */
+ public void connect() throws Exception {
+ // Intialize Hash
+ intializeHash(DrupalServiceCommands.SystemConnect);
- }
+ // Intialize Service
+ intializeService();
- public boolean connect() {
- try {
+ Debug.println("XmlService Service Intialized", " ");
- //Intialize Hash
- intializeHash(DrupalServiceCommands.SystemConnect);
+ try {
+ HashMap map = (HashMap) XmlService
+ .execute(DrupalServiceCommands.SystemConnect.toString(),
+ new Object[] {});
+
+ this.SessionID = (String) map.get("sessid");
- //Intialize Service
- IntializeService();
+ log.finest("Conn SessionID: " + this.SessionID);
+ } catch (Exception x) {
+ throw new Exception("cannot connect to "+ServiceURL+": "+x.getMessage(),x);
+ }
- Debug.println("XmlService Service Intialized"," ");
- HashMap map = (HashMap)XmlService.execute(DrupalServiceCommands.SystemConnect.toString(), new Object[]{});
+ }
- this.SessionID = (String)map.get("sessid");
+ public boolean login(String username, String password) {
+ try {
+ intializeHash(DrupalServiceCommands.UserLogin);
- Debug.println("Conn SessionID: ", this.SessionID);
+ Vector params = getDefaultParams();
+ // Add Login Paramaters
+ params.add(username);
+ params.add(password);
- return true;
+ HashMap o = (HashMap) XmlService.execute(
+ DrupalServiceCommands.UserLogin.toString(), params);
+ if (!o.isEmpty()) {
+ if (o.containsKey(username) && o.containsKey(password)) {
+ // confirmLogin(HashMap loginValue, String username, String
+ // password);
+ }
+ }
+ this.SessionID = (String) o.get("sessid");
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }
+ Debug.println("Successfull Login:", o.toString());
+ return true;
+ } catch (Exception e) {
+ log.log(Level.WARNING, e.getMessage(), e);
+ }
+ return false;
+ }
- public boolean login(String username, String password) {
- try {
- intializeHash(DrupalServiceCommands.UserLogin);
+ /*
+ * public void Login(String username, String password) { try {
+ * IntializeHash(DrupalServiceCommands.UserLogin);
+ *
+ *
+ * Vector params = GetDefaultParams(); //Add Login Paramaters
+ * params.add(username); params.add(password);
+ *
+ * HashMap o =
+ * (HashMap)XmlService.execute(DrupalServiceCommands.UserLogin.toString(),
+ * params);
+ *
+ * this.SessionID = (String)o.get("sessid");
+ *
+ * Debug.println("Successfull Login:", o.toString()); } catch (Exception e) {
+ * System.out.println(e.toString()); } }
+ */
- Vector params = GetDefaultParams();
- //Add Login Paramaters
- params.add(username);
- params.add(password);
+ public boolean logout() throws Exception {
+ intializeHash(DrupalServiceCommands.UserLogout);
+ Vector params = getDefaultParams();
+ params.add(this.SessionID);
+ Object o = XmlService.execute(DrupalServiceCommands.UserLogout
+ .toString(), params);
- HashMap o = (HashMap)XmlService.execute(DrupalServiceCommands.UserLogin.toString(), params);
- if(!o.isEmpty()) {
- if(o.containsKey(username) && o.containsKey(password)) {
- //confirmLogin(HashMap loginValue, String username, String password);
- }
- }
- this.SessionID = (String)o.get("sessid");
+ Debug.println("Logout Sucessfull:", o.toString());
+ return true;
+ }
- Debug.println("Successfull Login:", o.toString());
- return true;
- } catch (Exception e) {
- e.printStackTrace();
- }
- return false;
- }
- /*
- public void Login(String username, String password) {
- try {
- IntializeHash(DrupalServiceCommands.UserLogin);
+ // Testing New Things Not valid
+ public void testFileSave() throws Exception {
+ intializeHash(DrupalServiceCommands.FileSave);
+ byte[] Filebyte = new byte[10];
+ Vector params = getDefaultParams();
+ params.add(Filebyte);
+ Object o = XmlService.execute(DrupalServiceCommands.FileSave
+ .toString(), params);
+ Debug.println("Test Sucessfull:", o.toString());
+ }
- Vector params = GetDefaultParams();
- //Add Login Paramaters
- params.add(username);
- params.add(password);
-
- HashMap o = (HashMap)XmlService.execute(DrupalServiceCommands.UserLogin.toString(), params);
-
- this.SessionID = (String)o.get("sessid");
-
- Debug.println("Successfull Login:", o.toString());
-
- } catch (Exception e) {
- System.out.println(e.toString());
- }
- }
-*/
-
- public boolean logout() {
-
- try {
- intializeHash(DrupalServiceCommands.UserLogout);
- Vector params = GetDefaultParams();
- params.add(this.SessionID);
- Object o = XmlService.execute(DrupalServiceCommands.UserLogout.toString(), params);
-
- Debug.println("Logout Sucessfull:",o.toString());
- return true;
- } catch (Exception e) {
- System.out.println(e.toString());
- return false;
- }
- }
-
- //Testing New Things Not valid
- public void testFileSave() {
- try {
-
- intializeHash(DrupalServiceCommands.FileSave);
- byte[] Filebyte = new byte[10];
- Vector params = GetDefaultParams();
- params.add(Filebyte);
- Object o = XmlService.execute(DrupalServiceCommands.FileSave.toString(), params);
- Debug.println("Test Sucessfull:", o.toString());
- } catch (Exception e) {
- System.out.println(e.toString());
- }
- }
-
public Object nodeSave(DrupalNode node) throws Exception {
- try {
- intializeHash(DrupalServiceCommands.NodeSave);
- Vector params = GetDefaultParams();
- params.add(node);
- Object o = XmlService.execute(DrupalServiceCommands.NodeSave.toString(), params);
- Debug.println("Test Sucessfull:", o.toString());
- return o;
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
-
+ intializeHash(DrupalServiceCommands.NodeSave);
+ Vector params = getDefaultParams();
+ params.add(node);
+ Object o = XmlService.execute(
+ DrupalServiceCommands.NodeSave.toString(), params);
+ Debug.println("Test Sucessfull:", o.toString());
+ return o;
+
}
}
\ No newline at end of file
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/ApertureServer.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/ApertureServer.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/ApertureServer.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -38,6 +38,7 @@
import org.ontoware.rdf2go.model.node.impl.URIImpl;
import org.semanticdesktop.aperture.datasource.DataSourceFactory;
import org.semanticdesktop.aperture.detector.DataSourceDescription;
+import org.semanticdesktop.aperture.server.configui.bean.DataSourceUIBean;
import org.semanticdesktop.aperture.vocabulary.NIE;
/**
@@ -526,6 +527,15 @@
throws ApertureServerException;
/**
+ * Get the log messages of this datasource
+ * @return previous log messages of a datasource, as recorded by the CrawlerHandler.
+ * @throws ApertureServerException if the source under the given URI
+ * has not been found or something else goes wrong
+ */
+ public DataSourceLog getDataSourceLog(String dataSourceUri)
+ throws ApertureServerException;
+
+ /**
* Returns the URI that was last crawled by this aperture server.
* @return The URI of a data item that was last crawled by this aperture
* server.
@@ -604,4 +614,10 @@
* @return true, if the user is an authenticated administrator
*/
public boolean authenticateAdministrator(String username, String password);
+
+ /**
+ * @param dataSourceUri the uri of the data source to be retrieved.
+ * @return A single data source bean.
+ */
+ public DataSourceUIBean getDataSourceBean(String dataSourceUri);
}
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/DataSourceLog.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/DataSourceLog.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/DataSourceLog.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -3,14 +3,19 @@
*/
package org.semanticdesktop.aperture.server;
+import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.Collection;
import java.util.LinkedList;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
@@ -46,8 +51,21 @@
*
* So the solution for this problem, using off-the-shelf software, is to use the JUL logger
* and the XMLFormatter, as this log format is at least documented and parsable.
- * I have to write the parser myself, though.
+ * BUT: Oh pots of gold, there is no parser for the "standard" xml format created by JUL.
+ * How standard is that?
+ * I have to write the parser myself.
*
+ * BUT the XMLFormatter is from hell and parsing XML files the same. the output format throws an error:
+ * "xml document structures must start and end with the same entity",
+ * probably because the XMLFormatter didn't close() yet.
+ * So XML is not the right format, its not standard and I can't parse it.
+ * Hence see with what I had to come up with.
+ * <code>
+ * LEVEL;MILLIS;"MESSAGE"
+ * </code>
+ *
+ * replaces: \n with \\n, " with ', \r with nothing.
+ *
* Note: I tried to expose only a slf4j logger, and use the JUL Logger only internally,
* but the bastards from SLF4J made the constructor new JDK14LoggerAdapter(loggerJUL) private,
* so this "oh so standardized and properly programmed" library cannot be used to wrap a JUL logger.
@@ -62,7 +80,7 @@
*/
static final Level TRESHOLD = Level.CONFIG;
- Logger loggerJUL;
+ final Logger loggerJUL;
/**
* String identifying the datasource to which this log belongs
@@ -77,51 +95,78 @@
/**
* Output log formatter
*/
- XMLFormatter xmlformatter;
+ Formatter formatter = new ApertureDSLogFormatter();
+
/**
* output log file handler
*/
- StreamHandler xmlfilehandler;
+ StreamHandler filestreamhandler;
/**
- * A printstream to autoflush, because the xmlfilestream does not have this
+ * A printstream to autoflush, because the filestreamhandler does not have this
*/
- PrintStream xmlfilestreamflusher;
+ PrintStream filestreamflusher;
/**
* the file to write to
*/
- FileOutputStream xmlfilestream;
+ FileOutputStream filestream;
- /**
- * A dummy entity resolver to get around the problem that every xml parsing
- * involves dereferencing DTDs
- */
- final static EntityResolver dummyEntityResolver = new EntityResolver() {
- public InputSource resolveEntity(String publicID, String systemID)
- throws SAXException {
- return new InputSource(new StringReader(""));
- }
- };
+
+ private static class ApertureDSLogFormatter extends Formatter {
+
+ @Override
+ public String format(LogRecord record) {
+ String msg = record.getMessage();
+ if (msg==null)
+ msg = "";
+ else
+ {
+ msg.replace("\n", "\\n");
+ msg.replace("\r", ""); // windows CR+LF
+ msg.replace("\"", "'");
+ }
+ String result =
+ record.getLevel().getName() + ";" +
+ record.getMillis()+";\""+msg+"\"" +
+ "\n";
+ ;
+
+ return result;
+ }
+
+ }
+
+ static class FlushingFileStreamHandler extends StreamHandler {
+
+ public FlushingFileStreamHandler(OutputStream out, Formatter formatter) {
+ super(out, formatter);
+ }
+
+ /**
+ * yep, its ugly, but flushing from the outside of this would be even worse.
+ */
+ @Override
+ public synchronized void publish(LogRecord record) {
+ super.publish(record);
+ flush();
+ }
+
+ }
+
+
+
+
/**
- * Oh pots of gold, there is no parser for the "standard" xml format created by JUL.
- * How standard is that?
- *
* This class has a serious flaw: when the log file gets BIG, it has to load the whole log
* file into memory, which is a bit stupid. I am not going to do that better, if you want,
* go ahead.
*
- * According to
- * <a href="http://java.sun.com/j2se/1.4.2/docs/guide/util/logging/overview.html#3.0">DTD</a>,
- * this is to do:
- * date, millis, sequence, logger?, level,
- * class?, method?, thread?, message, key?, catalog?, param*, exception?
*
* @author sauermann
- *
*/
- private class LogParser extends DefaultHandler {
+ private class LogParser {
@@ -130,12 +175,6 @@
*/
@SuppressWarnings("unchecked")
final Collection buffer;
-
- LogRecord cur = null;
-
- StringBuffer curValue = null;
- String curException = null;
-
final Level minLevel;
public LogParser(int length, Level minLevel) {
@@ -149,118 +188,47 @@
@SuppressWarnings("unchecked")
public Collection<LogRecord> parse(File file) throws Exception {
- /*
- SAXParserFactory spf = SAXParserFactory.newInstance();
- spf.setValidating(false);
- spf.setXIncludeAware(false);
- SAXParser parser = spf.newSAXParser();
- parser.getXMLReader().setEntityResolver(dummyEntityResolver);
- parser.getXMLReader().setContentHandler(this);
- //reader.setErrorHandler(new MyErrorHandler());
- parser.parse(file, this);
- */
- XMLReader reader = XMLReaderFactory.createXMLReader();
- reader.setErrorHandler(new ErrorHandler() {
-
- public void error(SAXParseException exception)
- throws SAXException {
- // TODO Auto-generated method stub
+
+ FileInputStream fin = new FileInputStream(file);
+ try {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fin));
+ String in;
+ do {
+ in = reader.readLine();
+ if (in==null)
+ continue;
- }
-
- public void fatalError(SAXParseException exception)
- throws SAXException {
- // TODO Auto-generated method stub
+ try {
+ LogRecord cur = new LogRecord(Level.ALL, "");
+
+ // parse line, yep, this is by hand
+ int i = in.indexOf(";");
+ Level lev = Level.parse(in.substring(0,i));
+ cur.setLevel(lev);
+ int j = in.indexOf(";", i+1);
+ long millis = Long.parseLong(in.substring(i+1, j));
+ cur.setMillis(millis);
+ if (in.charAt(j+1)!='\"')
+ throw new Exception("message must start with \"");
+ String msg = in.substring(j+2, in.length()-1);
+ cur.setMessage(msg);
+ buffer.add(cur);
+
+ } catch (Exception x) {
+ // swallow. it is important that something shows.
+ System.err.println("Error parsing line "+in);
+ x.printStackTrace();
+ }
- }
-
- public void warning(SAXParseException exception)
- throws SAXException {
- // TODO Auto-generated method stub
- }
-
- });
- reader.setEntityResolver(dummyEntityResolver);
- reader.setContentHandler(this);
- FileInputStream fin = new FileInputStream(file);
- try {
- reader.parse(new InputSource(fin));
+
+ } while (in != null);
} finally {
fin.close();
}
return (Collection<LogRecord>)buffer;
}
-
- @Override
- public void endElement(String uri, String localName, String name)
- throws SAXException {
- if ("record".equals(localName)) {
- // add the current element to the buffer
- if (cur == null)
- throw new SAXException("closing 'record' was missing an opening 'record'");
- // only return those above the required log level
- if (cur.getLevel().intValue() >= minLevel.intValue())
- buffer.add(cur);
- cur = null;
- curValue = null;
- curException = null;
- } else {
- if (curValue == null) // empty
- return;
- if (cur == null)
- throw new SAXException("handling close of '"+localName+"': missing open element 'record'");
-
- // trim curValue, this is always a problem with newlines
- String val = curValue.toString().trim();
-
- // "date" - not parse it, its not in the logrecord
- // "sequence" - not
- // class?, method?, thread? - not
- // key?, catalog?, param* - not
- if ("millis".equals(localName)) {
- long millis = Long.parseLong(val);
- cur.setMillis(millis);
- } else if ("logger".equals(localName)) {
- cur.setLoggerName(val);
- } else if ("level".equals(localName)) {
- cur.setLevel(Level.parse(val));
- } else if ("message".equals(localName)) {
- // are we inside an exception?
- if (curException != null)
- curException = val;
- else
- cur.setMessage(val);
- } else if ("exception".equals(localName))
- curException = null;
-
- // the curValue was read, free it
- curValue = null;
- }
-
- }
-
- @Override
- public void characters(char[] ch, int start, int length)
- throws SAXException {
- if (curValue == null)
- curValue = new StringBuffer(new String(ch, start, length));
- else
- curValue.append(ch, start, length);
- }
-
- /**
- * Parse according to
- */
- @Override
- public void startElement(String uri, String localName, String name,
- Attributes attributes) throws SAXException {
- if ("record".equals(localName))
- cur = new LogRecord(Level.ALL, "");
- else if ("exception".equals(localName))
- curException = "";
- }
}
/**
@@ -275,25 +243,25 @@
this.logfile = logfile;
// create the JUL Logger
loggerJUL = Logger.getLogger(getClass().getName()+"."+dataSourceUri);
- // add handler
- xmlformatter = new XMLFormatter();
- xmlfilestream = new FileOutputStream(logfile, true);
- xmlfilestreamflusher = new PrintStream(xmlfilestream, true);
- xmlfilehandler = new StreamHandler(xmlfilestreamflusher, xmlformatter) {
-
- /**
- * yep, its ugly, but flushing from the outside of this would be even worse.
- */
- @Override
- public synchronized void publish(LogRecord record) {
- super.publish(record);
- flush();
- }
-
- };
- loggerJUL.addHandler(xmlfilehandler);
+ // NOTE: in rare occasions, this logger can be attached multiple times.
+ // For example, this happens when Tomcat restarts during debugging.
+ // Then its good to remove the old handler by other instances of this class,
+ // otherwise the messages are written twice.
+ // It should be taken care of by the finalizer, but who trusts that.
+ // Don't remove this - look up - the logger has its own name, containing the datasourceuri
+ // - so this is really only on weird occasions.
+ //
+ for (Handler handler: loggerJUL.getHandlers()) {
+ if (handler instanceof FlushingFileStreamHandler)
+ loggerJUL.removeHandler(handler);
+ }
+ // add our handler
+ filestream = new FileOutputStream(logfile, true);
+ filestreamflusher = new PrintStream(filestream, true);
+ filestreamhandler = new FlushingFileStreamHandler(filestreamflusher, formatter);
+ loggerJUL.addHandler(filestreamhandler);
loggerJUL.setLevel(TRESHOLD); // to be safe, set on both
- xmlfilehandler.setLevel(TRESHOLD); // to be safe, set on both
+ filestreamhandler.setLevel(TRESHOLD); // to be safe, set on both
}
@@ -322,13 +290,21 @@
@Override
protected void finalize() throws Throwable {
- if (xmlfilestream != null) {
+ if (filestreamhandler != null) {
+ loggerJUL.removeHandler(filestreamhandler);
+ filestreamhandler = null;
+ }
+ if (filestreamflusher != null) {
+ filestreamflusher.close();
+ filestreamflusher = null;
+ }
+ if (filestream != null) {
try {
- xmlfilestream.close();
+ filestream.close();
} catch (Exception x) {
// swallow
}
- xmlfilestream = null;
+ filestream = null;
}
super.finalize();
}
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/bean/DataSourceUIBean.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/bean/DataSourceUIBean.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/bean/DataSourceUIBean.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -50,6 +50,7 @@
import org.ontoware.rdf2go.model.Syntax;
import org.ontoware.rdf2go.model.node.URI;
import org.ontoware.rdf2go.vocabulary.RDF;
+import org.semanticdesktop.aperture.accessor.AccessData;
import org.semanticdesktop.aperture.rdf.RDFContainer;
import org.semanticdesktop.aperture.rdf.impl.RDFContainerImpl;
import org.semanticdesktop.aperture.server.CrawlerStateBean;
@@ -91,6 +92,11 @@
/** Last error message (or null if no error detected) */
private String lastErrorMessage;
+
+ /**
+ * only for reading the size.
+ */
+ private AccessData accessData;
////////////////////////////////////////////////////////////////////////
///////////////////////////// CONSTRUCTOR //////////////////////////////
@@ -104,6 +110,7 @@
* @param crawlerStateBean The bean containing the current state of this
* data source.
* @param syntax The serialization syntax used.
+ * @param accessData only for reading the size.
* @param ontologyDAO The ontologyDAO used to interact with the data source
* ontology.
* @throws DataWrapperUIException if the initialization fails
@@ -113,6 +120,7 @@
String configuration,
CrawlerStateBean crawlerStateBean,
Syntax syntax,
+ AccessData accessData,
DataSourceOntologyDAO ontologyDAO) throws DataWrapperUIException {
Model model = null;
@@ -139,6 +147,7 @@
"Couldn't initialize the data source bean",
ioe);
}
+ this.accessData = accessData;
}
////////////////////////////////////////////////////////////////////////
@@ -373,6 +382,59 @@
}
/**
+ * Statistics: get the size of the accessdata
+ * @return the size of the access data.
+ */
+ public int getAccessDataSize() {
+ return accessData.getSize();
+ }
+
+ /**
+ * Statistics: get the ids of the access datas
+ * @return the set of ids (Strings) from accessdata. Note that this can get massively large
+ */
+ public Set getAccessDataIDSet() {
+ return accessData.getStoredIDs();
+ }
+
+ /**
+ * Return the last access date for this ID
+ * @param id the ID to check
+ * @return the last access value. if the value is unparseable, 0
+ * @throws IndexOutOfBoundsException when the ID has no last access value
+ */
+ public long getAccessDataLastAccess(String id)
+ throws IndexOutOfBoundsException {
+ String val = accessData.get(id, AccessData.DATE_KEY);
+ if (val == null)
+ throw new IndexOutOfBoundsException("no last access found for "+id);
+ try {
+ return Long.parseLong(val);
+ } catch (Exception x) {
+ log.fine("cannot read value '"+val+"': "+x);
+ return 0;
+ }
+ }
+
+ /**
+ * Return the byte size for this ID
+ * @param id the ID to check
+ * @return the byte size. if the value is unparseable, 0
+ * @throws IndexOutOfBoundsException when the ID has no last access value
+ */
+ public long getAccessDataByteSize(String id)
+ throws IndexOutOfBoundsException {
+ String val = accessData.get(id, AccessData.BYTE_SIZE_KEY);
+ if (val == null)
+ throw new IndexOutOfBoundsException("no byte size found for "+id);
+ try {
+ return Long.parseLong(val);
+ } catch (Exception x) {
+ log.fine("cannot read value '"+val+"': "+x);
+ return 0;
+ }
+ }
+ /**
* Releases all system resources owned by this data source ui bean.
*/
public void dispose() {
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/dao/DataSourceInstanceDAO.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/dao/DataSourceInstanceDAO.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/configui/dao/DataSourceInstanceDAO.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -96,18 +96,7 @@
for (String uri : uriList) {
try {
- String configuration = wrapper.getDataSourceConfig(
- uri,
- Syntax.Ntriples.getMimeType());
- CrawlerStateBean crawlerStateBean = wrapper
- .getDetailedCrawlerState(uri);
- DataSourceUIBean bean = new DataSourceUIBean(
- uri,
- configuration,
- crawlerStateBean,
- Syntax.Ntriples,
- ontologyDAO);
- result.add(bean);
+ result.add(wrapper.getDataSourceBean(uri));
} catch (Exception e) {
log.log(Level.SEVERE,"Couldn't create a data source bean",e);
}
@@ -120,23 +109,7 @@
* @return A single data source bean.
*/
public DataSourceUIBean getDataSourceBean(String dataSourceUri) {
- DataSourceUIBean bean = null;
- try {
- String configuration = wrapper.getDataSourceConfig(
- dataSourceUri,
- Syntax.Ntriples.getMimeType());
- CrawlerStateBean crawlerStateBean = wrapper
- .getDetailedCrawlerState(dataSourceUri);
- bean = new DataSourceUIBean(
- dataSourceUri,
- configuration,
- crawlerStateBean,
- Syntax.Ntriples,
- ontologyDAO);
- } catch (Exception e) {
- log.log(Level.SEVERE,"Couldn't create a data source bean",e);
- }
- return bean;
+ return wrapper.getDataSourceBean(dataSourceUri);
}
/**
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourceInformation.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourceInformation.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourceInformation.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -41,6 +41,7 @@
import org.semanticdesktop.aperture.crawler.Crawler;
import org.semanticdesktop.aperture.crawler.ExitCode;
import org.semanticdesktop.aperture.server.CrawlerStateBean;
+import org.semanticdesktop.aperture.server.DataSourceLog;
import org.semanticdesktop.aperture.server.STATE;
import org.semanticdesktop.aperture.server.impl.ServerImpl;
@@ -71,6 +72,16 @@
/** The path to a AccessData file */
private File accessDataFile;
+ /**
+ * The path to the logfile
+ */
+ private File logfile;
+
+ /**
+ * The data source logger
+ */
+ private DataSourceLog datasourcelog;
+
/** Timeout between two consecutive crawls */
private long timeout;
@@ -208,6 +219,10 @@
*/
public void setAccessDataFile(File accessDataFile) {
this.accessDataFile = accessDataFile;
+
+ // TODO: replace the whole configuration way. put all stuff of a datasource into one folder,
+ // including the config. this will make things a lot easier.
+ this.logfile = new File(accessDataFile.toString()+".logfile");
}
/** @return the interval between crawls */
@@ -263,4 +278,15 @@
throw new IOException("Wrong state of object after deserialisation");
}
}
+
+ /**
+ * Get the DataSourceLog. This can go wrong if the log file can't be created.
+ * @return the data source log file.
+ * @throws Exception if the file cannot be opened
+ */
+ public DataSourceLog getDataSourceLog() throws Exception {
+ if (datasourcelog == null)
+ datasourcelog = new DataSourceLog(uriString, logfile);
+ return datasourcelog;
+ }
}
Modified: aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourcePool.java
===================================================================
--- aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourcePool.java 2010-02-16 19:07:24 UTC (rev 2221)
+++ aperture-webserver/trunk/src/org/semanticdesktop/aperture/server/datasource/DataSourcePool.java 2010-02-16 22:25:57 UTC (rev 2222)
@@ -74,11 +74,11 @@
import org.semanticdesktop.aperture.server.ApertureServerException;
import org.semanticdesktop.aperture.server.CrawlerStateBean;
import org.semanticdesktop.aperture.server.DataSourceChangedEvent;
+import org.semanticdesktop.aperture.server.DataSourceLog;
import org.semanticdesktop.aperture.server.STATE;
import org.semanticdesktop.aperture.server.exception.DataSourceAlreadyPresentException;
import org.semanticdesktop.aperture.server.exception.DataSourceNotFoundException;
import org.semanticdesktop.aperture.server.exception.DataSourceNotLockedException;
-import org.semanticdesktop.aperture.server.impl.ServerCrawlerHandler;
import org.semanticdesktop.aperture.server.impl.ServerImpl;
import org.semanticdesktop.aperture.server.thread.CrawlingThreadPoo...
[truncated message content] |