Menu
▾
▴
[Babeldoc-devel] babeldoc/modules/scanner/src/com/babeldoc/scanner/worker HttpScanner.java, 1.8, 1.9 MailboxScanner.java, 1.34, 1.35
|
From: Santi B. <san...@us...> - 2007-08-04 11:38:03
|
Update of /cvsroot/babeldoc/babeldoc/modules/scanner/src/com/babeldoc/scanner/worker In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv1742/modules/scanner/src/com/babeldoc/scanner/worker Modified Files: HttpScanner.java MailboxScanner.java Log Message: 2007-08-04 Added readme/doc with javadoc generation 2007-03-15 Many changes: - MailboxScanner can filter by field TO - Can debug MailboxScanner - Add timeout connection parameter to MailboxScanner - MailboxScanner obtain the value of the properties: 'to', 'from' and 'reply to' - Add three parameters that they allowed to deny a regular expression from filters (to,from and reply) - Add encoding parameter to Pipeline Writefile - Pipeline HttpClient save cookie information - Add parameter to pipeline HttpClient. It indicates if we want copy attributes in the new document - Pipeline HttpClient can send files in the parameters of POST method. To Manage the MIME types automatically (from the extension of the file) when adding attached files - Names of attached files in SmtpWriterPipelineStage don't contains absolute path - When decompressing a file in DecompressionPipeline, the name and the extension of the decompressed file it's original name Index: HttpScanner.java =================================================================== RCS file: /cvsroot/babeldoc/babeldoc/modules/scanner/src/com/babeldoc/scanner/worker/HttpScanner.java,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** HttpScanner.java 24 Jul 2004 00:18:52 -0000 1.8 --- HttpScanner.java 4 Aug 2007 11:37:00 -0000 1.9 *************** *** 78,82 **** import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; - import org.apache.commons.httpclient.HttpRecoverableException; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.methods.GetMethod; --- 78,81 ---- *************** *** 97,101 **** * more and submit it. * ! * @author bmcdonald * @version $Revision$ */ --- 96,100 ---- * more and submit it. * ! * @author paspes * @version $Revision$ */ *************** *** 158,162 **** } } ! enqueue(method.getResponseBody(), attrs); method.releaseConnection(); } --- 157,165 ---- } } ! try{ ! enqueue(method.getResponseBody(), attrs); ! } catch (IOException ioe) { ! log.logError("scanner.HttpScanner.doScan.error",ioe); ! } method.releaseConnection(); } *************** *** 229,240 **** log.logDebug("Scanning: " + url); statusCode = httpClient.executeMethod(method); - } catch (HttpRecoverableException e) { - log.logWarn(I18n.get("scanner.HttpScanner.error.recoverable", - e.getMessage())); } catch (IOException e) { statusCode = -1; break; ! } } --- 232,245 ---- log.logDebug("Scanning: " + url); statusCode = httpClient.executeMethod(method); } catch (IOException e) { statusCode = -1; break; ! } catch (Exception e) { ! LogService.getInstance().logWarn( ! I18n.get( ! "scanner.HttpScanner.error.recoverable", ! e.getMessage())); ! } } Index: MailboxScanner.java =================================================================== RCS file: /cvsroot/babeldoc/babeldoc/modules/scanner/src/com/babeldoc/scanner/worker/MailboxScanner.java,v retrieving revision 1.34 retrieving revision 1.35 diff -C2 -d -r1.34 -r1.35 *** MailboxScanner.java 8 Apr 2004 00:18:16 -0000 1.34 --- MailboxScanner.java 4 Aug 2007 11:37:00 -0000 1.35 *************** *** 48,52 **** * $Header: * /cvsroot/babeldoc/babeldoc/modules/scanner/src/com/babeldoc/scanner/worker/MailboxScanner.java,v ! * 1.29.4.1 2003/12/10 09:48:34 hbenedict Exp $ $Date$ * $Author$ * --- 48,52 ---- * $Header: * /cvsroot/babeldoc/babeldoc/modules/scanner/src/com/babeldoc/scanner/worker/MailboxScanner.java,v ! * 1.4 2007/03/15 09:48:34 paspes () Exp $ $Date$ * $Author$ * *************** *** 55,61 **** --- 55,64 ---- import java.io.BufferedReader; + import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; + import java.io.FileOutputStream; import java.io.InputStream; import java.io.InputStreamReader; + import java.io.OutputStreamWriter; import java.util.ArrayList; import java.util.Collection; *************** *** 65,68 **** --- 68,72 ---- import javax.activation.MimeType; import javax.mail.Address; + import javax.mail.internet.InternetAddress; import javax.mail.BodyPart; import javax.mail.Flags; *************** *** 74,77 **** --- 78,82 ---- import javax.mail.Session; import javax.mail.Store; + import com.sun.mail.pop3.POP3Message; import com.babeldoc.core.I18n; import com.babeldoc.core.option.ConfigOption; *************** *** 86,90 **** * Mailbox scanner scans a mailbox for the existance of email messages to be forward to the pipeline. * ! * @author Bruce McDonald, Dejan Krsmanovic * @version $Revision$ */ --- 91,95 ---- * Mailbox scanner scans a mailbox for the existance of email messages to be forward to the pipeline. * ! * @author paspes * @version $Revision$ */ *************** *** 103,106 **** --- 108,113 ---- private String protocol; // pop3 + private int timeOut; // timeOut (ms) + private String username; // roger *************** *** 113,116 **** --- 120,129 ---- private boolean delete = false; + private boolean fromFilterResult; + + private boolean toFilterResult; + + private boolean subjectFilterResult; + /** Configuration options */ public static final String HOST = "host"; *************** *** 118,121 **** --- 131,136 ---- public static final String PROTOCOL = "protocol"; + public static final String TIME_OUT = "timeOut"; + public static final String FOLDER = "folder"; *************** *** 128,135 **** --- 143,162 ---- public static final String FROM_FILTER = "fromFilter"; + //Attribute that it indicates if the result of the waited for regular expression is true or false + //(it serves to deny regular expressions) + public static final String FROM_FILTER_RESULT = "fromFilterResult"; + public static final String TO_FILTER = "toFilter"; + //Attribute that it indicates if the result of the waited for regular expression is true or false + //(it serves to deny regular expressions) + public static final String TO_FILTER_RESULT = "toFilterResult"; + public static final String SUBJECT_FILTER = "subjectFilter"; + //Attribute that it indicates if the result of the waited for regular expression is true or false + //(it serves to deny regular expressions) + public static final String SUBJECT_FILTER_RESULT = "subjectFilterResult"; + public static final String DELETE = "deleteInvalid"; *************** *** 138,141 **** --- 165,170 ---- public static final String ATTACHMENT = "attachment"; + + public static final String BOTH = "both";// , paspes: get attachment and body //default values *************** *** 152,155 **** --- 181,189 ---- public static final String SCAN_TO_KEY = "to"; + + public static final String DISPOSITION_KEY = "disposition";// paspes + + public static final String MESSAGE_ID_KEY = "message_id";// paspes + public static final String MESSAGE_PART_SIZE_KEY = "size";// paspes:Bytes to message part *************** *** 170,184 **** Store store = null; Session session = null; try { Properties props = new Properties(); session = Session.getInstance(props, null); store = session.getStore(protocol); store.connect(host, username, password); folder = store.getFolder(folder_); folder.open(Folder.READ_WRITE); - Message[] messages = folder.getMessages(); - // Get the filter parameter values addFilter(FROM_FILTER); --- 204,236 ---- Store store = null; Session session = null; + boolean flagDeleteMsg = false; try { Properties props = new Properties(); + if(timeOut>0){ + if (getLog().isDebugEnabled()) { + getLog().logDebug("mail.pop3.timeout = " + timeOut +"ms"); + } + props.put("mail.pop3.connectiontimeout", ""+timeOut); + props.put("mail.pop3.timeout", ""+timeOut); + } + else{ + getLog().logDebug("mail.pop3.timeout Not defined"); + } + session = Session.getInstance(props, null); store = session.getStore(protocol); + // debug connexion + if (getLog().isDebugEnabled()) { + getLog().logDebug("store connecting... "); + } store.connect(host, username, password); + if (getLog().isDebugEnabled()) { + getLog().logDebug("connected. "); + } + folder = store.getFolder(folder_); folder.open(Folder.READ_WRITE); Message[] messages = folder.getMessages(); // Get the filter parameter values addFilter(FROM_FILTER); *************** *** 188,194 **** --- 240,250 ---- for (int i = 0; i < messages.length; i++) { try { + if (getLog().isDebugEnabled()) { + getLog().logDebug(" matchMessage " + i +" de "+ messages.length); + } if (matchMessage(messages[i])) { processMessage(messages[i]); deleteMessage(messages[i]); + flagDeleteMsg = true; } } catch (Exception e) { *************** *** 198,204 **** --- 254,264 ---- getLog().logInfo("Deleting message"); deleteMessage(messages[i]); + flagDeleteMsg = true; } } } + } catch (javax.mail.MessagingException me) { + throw new ScannerException( + "SocketTimeout ", me); } catch (Exception x) { throw new ScannerException( *************** *** 207,211 **** try { if (folder != null && folder.isOpen()) { ! folder.close(true); } if (store != null) { --- 267,278 ---- try { if (folder != null && folder.isOpen()) { ! // debug end connexion ! if (getLog().isDebugEnabled()) { ! getLog().logDebug("folder closing... "); ! } ! folder.close(flagDeleteMsg); ! if (getLog().isDebugEnabled()) { ! getLog().logDebug("closed. "); ! } } if (store != null) { *************** *** 227,230 **** --- 294,298 ---- host = this.getInfo().getStrValue(HOST); protocol = this.getInfo().getStrValue(PROTOCOL); + timeOut = this.getInfo().getIntValue(TIME_OUT); folder_ = this.getInfo().getStrValue(FOLDER); username = this.getInfo().getStrValue(USERNAME); *************** *** 234,237 **** --- 302,309 ---- subjectFilter = this.getInfo().getStrValue(SUBJECT_FILTER); delete = this.getInfo().getBooleanValue(DELETE); + fromFilterResult = this.getInfo().getBooleanValue(FROM_FILTER_RESULT); + toFilterResult = this.getInfo().getBooleanValue(TO_FILTER_RESULT); + subjectFilterResult = this.getInfo().getBooleanValue(SUBJECT_FILTER_RESULT); + //Dejan 03.10.2002. *************** *** 241,246 **** getFrom = BODY; } else { ! if (!getFrom.equalsIgnoreCase(BODY) ! && !getFrom.equalsIgnoreCase(ATTACHMENT)) { throw new ScannerConfigurationException( I18n.get("scanner.MailboxScanner.error.getFrom"), null); --- 313,318 ---- getFrom = BODY; } else { ! if (!getFrom.equalsIgnoreCase(BODY) && !getFrom.equalsIgnoreCase(ATTACHMENT) ! && !getFrom.equalsIgnoreCase(BOTH)) { throw new ScannerConfigurationException( I18n.get("scanner.MailboxScanner.error.getFrom"), null); *************** *** 254,259 **** //do nothing by now... } ! ! /** * Return the mail content from the part --- 326,330 ---- //do nothing by now... } ! /** * Return the mail content from the part *************** *** 276,282 **** StringBuffer result = new StringBuffer(); String line; ! while ((line = reader.readLine()) != null) { ! //System.out.println( "appending: "+line); result.append(line).append("\n"); } --- 347,353 ---- StringBuffer result = new StringBuffer(); String line; ! while ((line = reader.readLine()) != null) { ! //System.out.println( "!!!!!!!!!!!!!!!appending:"+line+":"); result.append(line).append("\n"); } *************** *** 324,329 **** * @see Message ScannerWorker */ ! private boolean matchMessage(Message m) throws javax.mail.MessagingException { ! boolean fromMatch = false; boolean toMatch = false; --- 395,400 ---- * @see Message ScannerWorker */ ! // private boolean matchMessage(Message m) throws javax.mail.MessagingException { ! private boolean matchMessage(Message m) throws Exception { boolean fromMatch = false; boolean toMatch = false; *************** *** 336,346 **** getLog().logDebug( "Testing " + from[j].toString() + " against " ! + getFilter(FROM_FILTER)); } ! if (acceptEntry(TO_FILTER, from[j].toString())) { fromMatch = true; } if (getLog().isDebugEnabled()) { ! getLog().logDebug("Result: " + new Boolean(fromMatch).toString()); } } --- 407,417 ---- getLog().logDebug( "Testing " + from[j].toString() + " against " ! + getFilter(FROM_FILTER) + " Testing sucessfuly, if result of test is: " + new Boolean(fromFilterResult).toString()); } ! if (acceptEntry(FROM_FILTER, from[j].toString())==fromFilterResult) { fromMatch = true; } if (getLog().isDebugEnabled()) { ! getLog().logDebug("Final result: " + new Boolean(fromMatch).toString()); } } *************** *** 348,374 **** // Check for a match in the recipients Address[] to = m.getRecipients(Message.RecipientType.TO); ! for (int j = 0; j < to.length; j++) { if (getLog().isDebugEnabled()) { getLog().logDebug( ! "Testing " + to[j].toString() + " against " + getFilter(TO_FILTER)); ! } ! if (acceptEntry(TO_FILTER, to[j].toString())) { ! toMatch = true; } if (getLog().isDebugEnabled()) { ! getLog().logDebug("Result: " + new Boolean(toMatch).toString()); } } - if (getLog().isDebugEnabled()) { - getLog().logDebug( - "Testing " + m.getSubject().toString() + " against " - + getFilter(SUBJECT_FILTER)); - } - subjectMatch = acceptEntry(SUBJECT_FILTER, m.getSubject()); - if (getLog().isDebugEnabled()) { - getLog().logDebug("Result: " + new Boolean(subjectMatch).toString()); - } - return (fromMatch && toMatch && subjectMatch); } --- 419,459 ---- // Check for a match in the recipients Address[] to = m.getRecipients(Message.RecipientType.TO); ! if(to==null){ ! getLog().logWarn("Testing TO null. Result false" ); ! return false; ! } ! else ! { ! for (int j = 0; j < to.length; j++) { ! if (getLog().isDebugEnabled()) { ! getLog().logDebug( ! "Testing " + to[j].toString() + " against " + getFilter(TO_FILTER) ! + " Testing sucessfuly, if result of test is: " + new Boolean(toFilterResult).toString()); ! } ! if (acceptEntry(TO_FILTER, to[j].toString())==toFilterResult) { ! toMatch = true; ! } ! if (getLog().isDebugEnabled()) { ! getLog().logDebug("Final result: " + new Boolean(toMatch).toString()); ! } ! } ! } ! ! if(m.getSubject()==null){ ! getLog().logWarn("Testing SUBJECT null. Result false" ); ! return false; ! } ! else{ if (getLog().isDebugEnabled()) { getLog().logDebug( ! "Testing " + m.getSubject().toString() + " against " ! + getFilter(SUBJECT_FILTER) + " Testing sucessfuly, if result of test is: " + new Boolean(subjectFilterResult).toString()); } + subjectMatch = (acceptEntry(SUBJECT_FILTER, m.getSubject())==subjectFilterResult); if (getLog().isDebugEnabled()) { ! getLog().logDebug("Final result: " + new Boolean(subjectMatch).toString()); } } return (fromMatch && toMatch && subjectMatch); } *************** *** 426,429 **** --- 511,539 ---- } + /** + * + * Put into string all address from Address[] + * + *@param Adr Address array Object + *@return string with the value of adr Object + * + */ + private String getAddressValues( Address[] adr){ + String ret = ""; + for (int j = 0; j < adr.length; j++) { + try{ + if(j==0) + ret = ((InternetAddress) adr[j]).getAddress(); + else + ret = "," + ((InternetAddress) adr[j]).getAddress(); + }catch(Exception e){ + if(j==0) + ret = adr[j].toString(); + else + ret = "," + adr[j].toString(); + } + } + return ret; + } /** *************** *** 436,454 **** private void processPart(Part part, Message originalMessage) throws Exception { String disposition = part.getDisposition(); String path = protocol + "://" + username + "@" + host + "/" + folder_; Long date = new Long(originalMessage.getSentDate().getTime()); //store all attributes Map attrs = new HashMap(); attrs.put(SCAN_DATE_KEY, date); attrs.put(SCAN_PATH_KEY, path); attrs.put(SCAN_SUBJECT_KEY, originalMessage.getSubject()); ! attrs.put(SCAN_FROM_KEY, originalMessage.getFrom()); ! attrs.put(SCAN_REPLYTO_KEY, originalMessage.getReplyTo()); ! ! attrs.put(SCAN_TO_KEY , originalMessage.getAllRecipients()); //attrs.put(SCAN_TO_KEY, originalMessage.getSubject()); if (disposition == null) { //Don't getChild message body if getFrom is attachment if (getFrom.equalsIgnoreCase(ATTACHMENT)) { --- 546,577 ---- private void processPart(Part part, Message originalMessage) throws Exception { String disposition = part.getDisposition(); + if (getLog().isDebugEnabled()) { + getLog().logDebug("MailboxScanner.processPart Disposition: " + disposition +"."); + } String path = protocol + "://" + username + "@" + host + "/" + folder_; Long date = new Long(originalMessage.getSentDate().getTime()); //store all attributes + // Map attrs = new HashMap(); attrs.put(SCAN_DATE_KEY, date); attrs.put(SCAN_PATH_KEY, path); attrs.put(SCAN_SUBJECT_KEY, originalMessage.getSubject()); ! attrs.put(SCAN_FROM_KEY, getAddressValues(originalMessage.getFrom())); ! attrs.put(SCAN_REPLYTO_KEY, getAddressValues(originalMessage.getReplyTo())); ! attrs.put(SCAN_TO_KEY , getAddressValues(originalMessage.getAllRecipients())); ! if(protocol.equals("pop3")){ ! try{ ! attrs.put(MESSAGE_ID_KEY ,((POP3Message) originalMessage).getMessageID()); ! }catch(Exception ex){ ! getLog().logError("Malformed POP3Message:", ex); ! } ! } //attrs.put(SCAN_TO_KEY, originalMessage.getSubject()); if (disposition == null) { + attrs.put(SCAN_TO_KEY , getAddressValues(originalMessage.getAllRecipients())); + attrs.put(DISPOSITION_KEY ,BODY ); + //Don't getChild message body if getFrom is attachment if (getFrom.equalsIgnoreCase(ATTACHMENT)) { *************** *** 464,474 **** //TODO: This should be done more clever then this attrs.put("contentType", "text/plain"); attrs.put(ScannerWorker.FILE_NAME_KEY, originalMessage.getSubject()); //feed document this.enqueue(data, attrs); } } else { //This is an attachment ! if (getFrom.equalsIgnoreCase(ATTACHMENT)) { byte[] content = null; if (this.isBinary()) { --- 587,610 ---- //TODO: This should be done more clever then this attrs.put("contentType", "text/plain"); + //Paspes::Paspes: Save original charset encoding + //Paspes:: String encodingCharset = new MimeType(part.getContentType()).getParameter("charset"); + //Paspes::if (getLog().isDebugEnabled()) { + //Paspes:: getLog().logDebug("MailboxScanner.processPart encoding:" + encodingCharset); + //Paspes::} attrs.put(ScannerWorker.FILE_NAME_KEY, originalMessage.getSubject()); + attrs.put(MESSAGE_PART_SIZE_KEY, new Integer(data.length)); + + if (getLog().isDebugEnabled()) { + getLog().logDebug("MailboxScanner.processPart enqueue body"); + } //feed document + //Paspes:: this.enqueue(data, attrs,encodingCharset); this.enqueue(data, attrs); } } else { + //This is an attachment ! if (getFrom.equalsIgnoreCase(ATTACHMENT) || getFrom.equalsIgnoreCase(BOTH)) { ! attrs.put(DISPOSITION_KEY ,ATTACHMENT ); byte[] content = null; if (this.isBinary()) { *************** *** 479,482 **** --- 615,623 ---- } attrs.put(ScannerWorker.FILE_NAME_KEY, part.getFileName()); + attrs.put(MESSAGE_PART_SIZE_KEY, new Integer(content.length)); + if (getLog().isDebugEnabled()) { + getLog().logDebug("MailboxScanner.processPart enqueue attachment"); + } + // Paspes::this.enqueue(content, attrs,null); this.enqueue(content, attrs); } *************** *** 531,534 **** --- 672,679 ---- I18n.get("scanner.MailboxScannerInfo.option.protocol"))); + options.add(new ConfigOption(MailboxScanner.TIME_OUT, + IConfigOptionType.INTEGER,"0", false, + I18n.get("scanner.MailboxScannerInfo.option.timeOut"))); + options.add(new ConfigOption(MailboxScanner.FOLDER, IConfigOptionType.STRING, MailboxScanner.DEFAULT_FOLDER, false, *************** *** 560,563 **** --- 705,720 ---- I18n.get("scanner.MailboxScannerInfo.option.subjectFilter"))); + options.add(new ConfigOption(MailboxScanner.FROM_FILTER_RESULT, + IConfigOptionType.BOOLEAN, "true", false, + I18n.get("scanner.MailboxScannerInfo.option.filterResult"))); + + options.add(new ConfigOption(MailboxScanner.TO_FILTER_RESULT, + IConfigOptionType.BOOLEAN, "true", false, + I18n.get("scanner.MailboxScannerInfo.option.filterResult"))); + + options.add(new ConfigOption(MailboxScanner.SUBJECT_FILTER_RESULT, + IConfigOptionType.BOOLEAN, "true", false, + I18n.get("scanner.MailboxScannerInfo.option.filterResult"))); + options.add(new ConfigOption(MailboxScanner.DELETE, IConfigOptionType.BOOLEAN, "false", true, *************** *** 566,569 **** return options; } ! } ! --- 723,725 ---- return options; } ! } \ No newline at end of file |
