Menu
▾
▴
[jtcfrost-cvs] frost-wot/source/frost/threads FileListUploadThread.java, 1.9, 1.10 FileRequestsThread.java, 1.12, 1.13 FilePointersThread.java, 1.13, 1.14 MessageThread.java, 1.19, 1.20
|
From: Mailing l. f. C. c. m. <jtc...@li...> - 2007-06-28 13:27:18
|
Update of /cvsroot/jtcfrost/frost-wot/source/frost/threads In directory sc8-pr-cvs6.sourceforge.net:/tmp/cvs-serv16092/source/frost/threads Modified Files: FileListUploadThread.java FileRequestsThread.java FilePointersThread.java MessageThread.java Log Message: - removed import from non-database versions (previous version must be at least 23-Dec-2006 - migrating vital internal tables from McKoi SQL database to perst object database, much faster, lesser usage of disk space and memory (messages are still in in McKoi database) - archiving and cleanup of McKoi tables only all 5 days now - fixed message download bug, it happened that Frost stopped to download message indices too early - fixed another message download bug, not all days backwards were correctly downloaded - Frost now receives only messages from Frost version 20-Jun-2006 or later (the message must have a valid unique message id!) Index: FileListUploadThread.java =================================================================== RCS file: /cvsroot/jtcfrost/frost-wot/source/frost/threads/FileListUploadThread.java,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** FileListUploadThread.java 12 Apr 2007 08:50:16 -0000 1.9 --- FileListUploadThread.java 28 Jun 2007 13:26:39 -0000 1.10 *************** *** 24,27 **** --- 24,28 ---- import frost.fcp.*; import frost.fileTransfer.*; + import frost.storage.perst.*; import frost.util.*; Index: FileRequestsThread.java =================================================================== RCS file: /cvsroot/jtcfrost/frost-wot/source/frost/threads/FileRequestsThread.java,v retrieving revision 1.12 retrieving revision 1.13 diff -C2 -d -r1.12 -r1.13 *** FileRequestsThread.java 21 Apr 2007 11:54:39 -0000 1.12 --- FileRequestsThread.java 28 Jun 2007 13:26:39 -0000 1.13 *************** *** 28,32 **** import frost.fcp.*; import frost.fileTransfer.*; ! import frost.storage.database.applayer.*; import frost.transferlayer.*; import frost.util.*; --- 28,32 ---- import frost.fcp.*; import frost.fileTransfer.*; ! import frost.storage.perst.*; import frost.transferlayer.*; import frost.util.*; *************** *** 43,47 **** private static final int sleepTime = 10 * 60 * 1000; - private GlobalIndexSlotsDatabaseTable indexSlots; private String keyPrefix; --- 43,46 ---- *************** *** 50,55 **** private FileRequestsThread() { - indexSlots = new GlobalIndexSlotsDatabaseTable(GlobalIndexSlotsDatabaseTable.REQUESTS); - String fileBase = Core.frostSettings.getValue(SettingsClass.FILE_BASE); keyPrefix = "KSK@frost/filerequests/" + fileBase + "-"; --- 49,52 ---- *************** *** 67,71 **** * Returns true if no error occured. */ ! private boolean uploadRequestFile(String dateStr, long sqlDate) throws Throwable { // get a list of CHK keys to send --- 64,68 ---- * Returns true if no error occured. */ ! private boolean uploadRequestFile(String dateStr, IndexSlot gis) throws Throwable { // get a list of CHK keys to send *************** *** 93,97 **** String insertKey = keyPrefix + dateStr + "-"; ! boolean wasOk = GlobalFileUploader.uploadFile(indexSlots, sqlDate, tmpRequestFile, insertKey, ".xml", true); tmpRequestFile.delete(); System.out.println("uploadRequestFile: upload finished, wasOk="+wasOk); --- 90,94 ---- String insertKey = keyPrefix + dateStr + "-"; ! boolean wasOk = GlobalFileUploader.uploadFile(gis, tmpRequestFile, insertKey, ".xml", true); tmpRequestFile.delete(); System.out.println("uploadRequestFile: upload finished, wasOk="+wasOk); *************** *** 102,106 **** } ! private void downloadDate(String dateStr, long sqlDate, boolean isForToday) throws Throwable { // "KSK@frost/filerequests/2006.11.1-<index>.xml" --- 99,103 ---- } ! private void downloadDate(String dateStr, IndexSlot gis, boolean isForToday) throws Throwable { // "KSK@frost/filerequests/2006.11.1-<index>.xml" *************** *** 113,117 **** maxFailures = 2; // skip a maximum of 1 empty slot for backload } ! int index = indexSlots.findFirstDownloadSlot(sqlDate); int failures = 0; --- 110,114 ---- maxFailures = 2; // skip a maximum of 1 empty slot for backload } ! int index = gis.findFirstDownloadSlot(); int failures = 0; *************** *** 128,134 **** if( result == null ) { // download failed. ! failures++; // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, sqlDate); continue; } --- 125,134 ---- if( result == null ) { // download failed. ! if( gis.isDownloadIndexBehindLastSetIndex(index) ) { ! // we stop if we tried maxFailures indices behind the last known index ! failures++; ! } // next loop we try next index ! index = gis.findNextDownloadSlot(index); continue; } *************** *** 140,151 **** System.out.println("FileRequestsThread.downloadDate: Skipping index "+index+" for now, will try again later."); // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, sqlDate); continue; } // downloaded something, mark it ! indexSlots.setDownloadSlotUsed(index, sqlDate); // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, sqlDate); if( result.getErrorCode() == GlobalFileDownloaderResult.ERROR_FILE_TOO_BIG ) { --- 140,151 ---- System.out.println("FileRequestsThread.downloadDate: Skipping index "+index+" for now, will try again later."); // next loop we try next index ! index = gis.findNextDownloadSlot(index); continue; } // downloaded something, mark it ! gis.setDownloadSlotUsed(index); // next loop we try next index ! index = gis.findNextDownloadSlot(index); if( result.getErrorCode() == GlobalFileDownloaderResult.ERROR_FILE_TOO_BIG ) { *************** *** 195,205 **** long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); ! // String dateStr = DateFun.getDate(i); ! // java.sql.Date sqlDate = DateFun.getSqlDateGMTDaysAgo(i); System.out.println("FileRequestsThread: starting download for "+dateStr); // download file pointer files for this date if( !isInterrupted() ) { ! downloadDate(dateStr, date, isForToday); } --- 195,205 ---- long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); ! IndexSlot gis = IndexSlotsStorage.inst().getSlotForDate( ! IndexSlotsStorage.REQUESTS, date); System.out.println("FileRequestsThread: starting download for "+dateStr); // download file pointer files for this date if( !isInterrupted() ) { ! downloadDate(dateStr, gis, isForToday); } *************** *** 208,217 **** try { System.out.println("FileRequestsThread: starting upload for "+dateStr); ! uploadRequestFile(dateStr, date); } catch(Throwable t) { logger.log(Level.SEVERE, "Exception catched during uploadRequestFile()", t); } } ! if( isInterrupted() ) { break; --- 208,219 ---- try { System.out.println("FileRequestsThread: starting upload for "+dateStr); ! uploadRequestFile(dateStr, gis); } catch(Throwable t) { logger.log(Level.SEVERE, "Exception catched during uploadRequestFile()", t); } } ! ! IndexSlotsStorage.inst().storeSlot(gis); ! if( isInterrupted() ) { break; *************** *** 230,238 **** break; } System.out.println("FileRequestsThread: sleeping 10 minutes"); Mixed.wait(sleepTime); // sleep 10 minutes } - - indexSlots.close(); } } --- 232,241 ---- break; } + + IndexSlotsStorage.inst().commitStore(); // commit changes for this run + System.out.println("FileRequestsThread: sleeping 10 minutes"); Mixed.wait(sleepTime); // sleep 10 minutes } } } Index: MessageThread.java =================================================================== RCS file: /cvsroot/jtcfrost/frost-wot/source/frost/threads/MessageThread.java,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** MessageThread.java 19 Apr 2007 15:11:23 -0000 1.19 --- MessageThread.java 28 Jun 2007 13:26:39 -0000 1.20 *************** *** 30,34 **** import frost.identities.*; import frost.messages.*; ! import frost.storage.database.applayer.*; import frost.transferlayer.*; import frost.util.*; --- 30,34 ---- import frost.identities.*; import frost.messages.*; ! import frost.storage.perst.*; import frost.transferlayer.*; import frost.util.*; *************** *** 43,48 **** private boolean downloadToday; - private IndexSlotsDatabaseTable indexSlots; - private static final Logger logger = Logger.getLogger(MessageThread.class.getName()); --- 43,46 ---- *************** *** 52,57 **** this.board = boa; this.maxMessageDownload = maxmsgdays; - - this.indexSlots = new IndexSlotsDatabaseTable(IndexSlotsDatabaseTable.MESSAGES, board); } --- 50,53 ---- *************** *** 82,86 **** if (isInterrupted()) { ! indexSlots.close(); notifyThreadFinished(this); return; --- 78,82 ---- if (isInterrupted()) { ! IndexSlotsStorage.inst().commitStore(); notifyThreadFinished(this); return; *************** *** 88,97 **** LocalDate localDate = new LocalDate(DateTimeZone.UTC); if (this.downloadToday) { // download only current date ! downloadDate(localDate); // after update check if there are messages for upload and upload them ! uploadMessages(); // does'nt get a message when message upload is disabled } else { // download up to maxMessages days to the past --- 84,96 ---- LocalDate localDate = new LocalDate(DateTimeZone.UTC); + long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); + + IndexSlot gis = IndexSlotsStorage.inst().getSlotForDate(board.getPrimaryKey(), date); if (this.downloadToday) { // download only current date ! downloadDate(localDate, gis); // after update check if there are messages for upload and upload them ! uploadMessages(gis); // doesn't get a message when message upload is disabled } else { // download up to maxMessages days to the past *************** *** 99,103 **** while (!isInterrupted() && daysBack < maxMessageDownload) { daysBack++; ! downloadDate(localDate.minusDays(daysBack)); } // after a complete backload run, remember finish time. --- 98,105 ---- while (!isInterrupted() && daysBack < maxMessageDownload) { daysBack++; ! localDate = localDate.minusDays(1); ! date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); ! gis = IndexSlotsStorage.inst().getSlotForDate(board.getPrimaryKey(), date); ! downloadDate(localDate, gis); } // after a complete backload run, remember finish time. *************** *** 107,115 **** } } logger.info("TOFDN: " + tofType + " Thread stopped for board " + board.getName()); } catch (Throwable t) { logger.log(Level.SEVERE, Thread.currentThread().getName() + ": Oo. Exception in MessageDownloadThread:", t); } ! indexSlots.close(); notifyThreadFinished(this); } --- 109,120 ---- } } + + IndexSlotsStorage.inst().storeSlot(gis); + logger.info("TOFDN: " + tofType + " Thread stopped for board " + board.getName()); } catch (Throwable t) { logger.log(Level.SEVERE, Thread.currentThread().getName() + ": Oo. Exception in MessageDownloadThread:", t); } ! IndexSlotsStorage.inst().commitStore(); notifyThreadFinished(this); } *************** *** 145,156 **** } ! protected void downloadDate(LocalDate localDate) throws SQLException { - String dirdate = DateFun.FORMAT_DATE.print(localDate); - long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); - int index = -1; int failures = 0; ! int maxFailures = 2; // skip a maximum of 2 empty slots while (failures < maxFailures) { --- 150,160 ---- } ! protected void downloadDate(LocalDate localDate, IndexSlot gis) throws SQLException { ! ! final String dirdate = DateFun.FORMAT_DATE.print(localDate); int index = -1; int failures = 0; ! int maxFailures = 2; // skip a maximum of 2 empty slots at the end of known indices while (failures < maxFailures) { *************** *** 161,167 **** if( index < 0 ) { ! index = indexSlots.findFirstDownloadSlot(date); } else { ! index = indexSlots.findNextDownloadSlot(index, date); } --- 165,171 ---- if( index < 0 ) { ! index = gis.findFirstDownloadSlot(); } else { ! index = gis.findNextDownloadSlot(index); } *************** *** 179,186 **** MessageDownloaderResult mdResult = MessageDownloader.downloadMessage(downKey, index, fastDownload, logInfo); - if( mdResult == null ) { // file not found ! failures++; continue; } --- 183,192 ---- MessageDownloaderResult mdResult = MessageDownloader.downloadMessage(downKey, index, fastDownload, logInfo); if( mdResult == null ) { // file not found ! if( gis.isDownloadIndexBehindLastSetIndex(index) ) { ! // we stop if we tried maxFailures indices behind the last known index ! failures++; ! } continue; } *************** *** 197,201 **** } ! indexSlots.setDownloadSlotUsed(index, date); if( mdResult.isFailure() ) { --- 203,207 ---- } ! gis.setDownloadSlotUsed(index); if( mdResult.isFailure() ) { *************** *** 245,248 **** --- 251,260 ---- } + // e.g. "E6936D085FC1AE75D43275161B50B0CEDB43716C1CE54E420F3C6FEB9352B462" (len=64) + if( mo.getMessageId() == null || mo.getMessageId().length() < 60 || mo.getMessageId().length() > 68 ) { + logger.log(Level.SEVERE, "Message has no unique message id - skipping Message: "+dirDate+";"+dateTime); + return false; + } + // ensure that time/date of msg is max. 1 day before/after dirDate DateMidnight dm = dateTime.toDateMidnight(); *************** *** 262,266 **** final String downloadingBoardName = b.getName().toLowerCase(); if( boardNameInMsg.equals(downloadingBoardName) == false ) { ! logger.log(Level.SEVERE, "Unequal boardnames - skipping message: "+mo.getBoardName().toLowerCase()+";"+b.getName().toLowerCase()); return false; } --- 274,278 ---- final String downloadingBoardName = b.getName().toLowerCase(); if( boardNameInMsg.equals(downloadingBoardName) == false ) { ! logger.log(Level.SEVERE, "Different boardnames - skipping message: "+mo.getBoardName().toLowerCase()+";"+b.getName().toLowerCase()); return false; } *************** *** 276,280 **** * Upload pending messages for this board. */ ! protected void uploadMessages() { FrostUnsentMessageObject unsendMsg = UnsentMessagesManager.getUnsentMessage(board); --- 288,292 ---- * Upload pending messages for this board. */ ! protected void uploadMessages(IndexSlot gis) { FrostUnsentMessageObject unsendMsg = UnsentMessagesManager.getUnsentMessage(board); *************** *** 301,305 **** UnsentMessagesManager.incRunningMessageUploads(); ! uploadMessage(unsendMsg, recipient); UnsentMessagesManager.decRunningMessageUploads(); --- 313,317 ---- UnsentMessagesManager.incRunningMessageUploads(); ! uploadMessage(unsendMsg, recipient, gis); UnsentMessagesManager.decRunningMessageUploads(); *************** *** 312,316 **** } ! private void uploadMessage(FrostUnsentMessageObject mo, Identity recipient) { logger.info("Preparing upload of message to board '" + board.getName() + "'"); --- 324,328 ---- } ! private void uploadMessage(FrostUnsentMessageObject mo, Identity recipient, IndexSlot gis) { logger.info("Preparing upload of message to board '" + board.getName() + "'"); *************** *** 354,359 **** senderId, this, ! indexSlots, ! now.toDateMidnight().getMillis(), MainFrame.getInstance(), board.getName()); --- 366,370 ---- senderId, this, ! gis, MainFrame.getInstance(), board.getName()); Index: FilePointersThread.java =================================================================== RCS file: /cvsroot/jtcfrost/frost-wot/source/frost/threads/FilePointersThread.java,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** FilePointersThread.java 21 Apr 2007 11:54:39 -0000 1.13 --- FilePointersThread.java 28 Jun 2007 13:26:39 -0000 1.14 *************** *** 28,32 **** import frost.fcp.*; import frost.fileTransfer.*; ! import frost.storage.database.applayer.*; import frost.transferlayer.*; import frost.util.*; --- 28,32 ---- import frost.fcp.*; import frost.fileTransfer.*; ! import frost.storage.perst.*; import frost.transferlayer.*; import frost.util.*; *************** *** 46,50 **** private static final int baseSleepTime = 15 * 60 * 1000; - private GlobalIndexSlotsDatabaseTable indexSlots; private String keyPrefix; --- 46,49 ---- *************** *** 53,57 **** private FilePointersThread() { - indexSlots = new GlobalIndexSlotsDatabaseTable(GlobalIndexSlotsDatabaseTable.FILELISTS); String fileBase = Core.frostSettings.getValue(SettingsClass.FILE_BASE); keyPrefix = "KSK@frost/filepointers/" + fileBase + "-"; --- 52,55 ---- *************** *** 69,73 **** * Returns true if no error occured. */ ! private boolean uploadIndexFile(String dateStr, long sqlDate) throws Throwable { // get a list of CHK keys to send --- 67,71 ---- * Returns true if no error occured. */ ! private boolean uploadIndexFile(String dateStr, IndexSlot gis) throws Throwable { // get a list of CHK keys to send *************** *** 104,108 **** String insertKey = keyPrefix + dateStr + "-"; System.out.println("uploadIndexFile: Starting upload of pointer file containing "+sharedFileCHKkeys.size()+" CHK keys to "+insertKey+"..."); ! boolean wasOk = GlobalFileUploader.uploadFile(indexSlots, sqlDate, tmpPointerFile, insertKey, ".xml", true); System.out.println("uploadIndexFile: upload finished, wasOk="+wasOk); tmpPointerFile.delete(); --- 102,106 ---- String insertKey = keyPrefix + dateStr + "-"; System.out.println("uploadIndexFile: Starting upload of pointer file containing "+sharedFileCHKkeys.size()+" CHK keys to "+insertKey+"..."); ! boolean wasOk = GlobalFileUploader.uploadFile(gis, tmpPointerFile, insertKey, ".xml", true); System.out.println("uploadIndexFile: upload finished, wasOk="+wasOk); tmpPointerFile.delete(); *************** *** 113,117 **** } ! private void downloadDate(String dateStr, long date, boolean isForToday) throws Throwable { // "KSK@frost/filelistpointer/2006.11.1-<index>.xml" --- 111,115 ---- } ! private void downloadDate(String dateStr, IndexSlot gis, boolean isForToday) throws Throwable { // "KSK@frost/filelistpointer/2006.11.1-<index>.xml" *************** *** 124,128 **** maxFailures = 2; // skip a maximum of 1 empty slot for backload } ! int index = indexSlots.findFirstDownloadSlot(date); int failures = 0; while (failures < maxFailures && index >= 0 ) { --- 122,126 ---- maxFailures = 2; // skip a maximum of 1 empty slot for backload } ! int index = gis.findFirstDownloadSlot(); int failures = 0; while (failures < maxFailures && index >= 0 ) { *************** *** 141,147 **** System.out.println("FilePointersThread.downloadDate: failure"); // download failed. ! failures++; // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, date); continue; } --- 139,148 ---- System.out.println("FilePointersThread.downloadDate: failure"); // download failed. ! if( gis.isDownloadIndexBehindLastSetIndex(index) ) { ! // we stop if we tried maxFailures indices behind the last known index ! failures++; ! } // next loop we try next index ! index = gis.findNextDownloadSlot(index); continue; } *************** *** 152,163 **** System.out.println("FilePointersThread.downloadDate: Skipping index "+index+" for now, will try again later."); // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, date); continue; } // downloaded something, mark it ! indexSlots.setDownloadSlotUsed(index, date); // next loop we try next index ! index = indexSlots.findNextDownloadSlot(index, date); if( result.getErrorCode() == GlobalFileDownloaderResult.ERROR_FILE_TOO_BIG ) { --- 153,165 ---- System.out.println("FilePointersThread.downloadDate: Skipping index "+index+" for now, will try again later."); // next loop we try next index ! index = gis.findNextDownloadSlot(index); continue; } // downloaded something, mark it ! gis.setDownloadSlotUsed(index); ! gis.modify(); // next loop we try next index ! index = gis.findNextDownloadSlot(index); if( result.getErrorCode() == GlobalFileDownloaderResult.ERROR_FILE_TOO_BIG ) { *************** *** 211,221 **** long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); ! // String dateStr = DateFun.getDate(i); ! // java.sql.Date sqlDate = DateFun.getSqlDateGMTDaysAgo(i); ! System.out.println("FilePointersThread: download for "+dateStr); // download file pointer files for this date if( !isInterrupted() ) { ! downloadDate(dateStr, date, isForToday); } --- 213,223 ---- long date = localDate.toDateMidnight(DateTimeZone.UTC).getMillis(); ! IndexSlot gis = IndexSlotsStorage.inst().getSlotForDate( ! IndexSlotsStorage.FILELISTS, date); ! System.out.println("FilePointersThread: download for "+dateStr); // download file pointer files for this date if( !isInterrupted() ) { ! downloadDate(dateStr, gis, isForToday); } *************** *** 224,228 **** try { System.out.println("FilePointersThread: upload for "+dateStr); ! uploadIndexFile(dateStr, date); } catch(Throwable t) { logger.log(Level.SEVERE, "Exception during uploadIndexFile()", t); --- 226,230 ---- try { System.out.println("FilePointersThread: upload for "+dateStr); ! uploadIndexFile(dateStr, gis); } catch(Throwable t) { logger.log(Level.SEVERE, "Exception during uploadIndexFile()", t); *************** *** 230,233 **** --- 232,237 ---- } + IndexSlotsStorage.inst().storeSlot(gis); + if( isInterrupted() ) { break; *************** *** 247,254 **** } // random sleeptime to anonymize our uploaded pointer files Mixed.waitRandom(baseSleepTime); } - indexSlots.close(); } } --- 251,259 ---- } + IndexSlotsStorage.inst().commitStore(); // commit changes for this run + // random sleeptime to anonymize our uploaded pointer files Mixed.waitRandom(baseSleepTime); } } } |