From: <th...@us...> - 2010-08-24 00:04:14
|
Revision: 1931 http://azsmrc.svn.sourceforge.net/azsmrc/?rev=1931&view=rev Author: the8472 Date: 2010-08-24 00:04:08 +0000 (Tue, 24 Aug 2010) Log Message: ----------- -filter out address bogons in many places -fix the calclation of the fetch queue Modified Paths: -------------- mldht/branches/indexer/DHTIndexer.jar mldht/branches/indexer/lbms/plugins/mldht/indexer/InfoHashGatherer.java mldht/branches/indexer/lbms/plugins/mldht/indexer/MetaDataGatherer.java mldht/branches/indexer/lbms/plugins/mldht/kad/DHT.java mldht/branches/indexer/lbms/plugins/mldht/kad/Database.java mldht/branches/indexer/lbms/plugins/mldht/kad/Node.java mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/NodeLookup.java mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/PeerLookupTask.java Added Paths: ----------- mldht/branches/indexer/lbms/plugins/mldht/kad/utils/AddressUtils.java Modified: mldht/branches/indexer/DHTIndexer.jar =================================================================== (Binary files differ) Modified: mldht/branches/indexer/lbms/plugins/mldht/indexer/InfoHashGatherer.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/indexer/InfoHashGatherer.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/indexer/InfoHashGatherer.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -60,7 +60,8 @@ while(keys.size() < MAX_BUFFER_SIZE && (toProcess = handoffQueue.poll()) != null) if(recentlyProcessed.get(toProcess) == null) // access recently processed entries to get LRU behavior keys.add(toProcess); - handoffQueue.clear(); + if(keys.size() >= MAX_BUFFER_SIZE) + handoffQueue.clear(); } } Modified: mldht/branches/indexer/lbms/plugins/mldht/indexer/MetaDataGatherer.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/indexer/MetaDataGatherer.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/indexer/MetaDataGatherer.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -194,7 +194,7 @@ currentPool = currentPool - targetPool; // scale max down by the percentage left until the buffer is full too - maxLookups = (int) (maxLookups * (1- currentPool/targetPool)); + maxLookups = (int) (maxLookups * (1.0- currentPool*1.0/targetPool)); targetNum = maxLookups - currentLookups; } Modified: mldht/branches/indexer/lbms/plugins/mldht/kad/DHT.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/DHT.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/DHT.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -27,18 +27,10 @@ import lbms.plugins.mldht.DHTConfiguration; import lbms.plugins.mldht.kad.Node.RoutingTableEntry; -import lbms.plugins.mldht.kad.messages.AnnounceRequest; -import lbms.plugins.mldht.kad.messages.AnnounceResponse; -import lbms.plugins.mldht.kad.messages.ErrorMessage; -import lbms.plugins.mldht.kad.messages.FindNodeRequest; -import lbms.plugins.mldht.kad.messages.FindNodeResponse; -import lbms.plugins.mldht.kad.messages.GetPeersRequest; -import lbms.plugins.mldht.kad.messages.GetPeersResponse; -import lbms.plugins.mldht.kad.messages.MessageBase; -import lbms.plugins.mldht.kad.messages.PingRequest; -import lbms.plugins.mldht.kad.messages.PingResponse; +import lbms.plugins.mldht.kad.messages.*; import lbms.plugins.mldht.kad.messages.ErrorMessage.ErrorCode; import lbms.plugins.mldht.kad.tasks.*; +import lbms.plugins.mldht.kad.utils.AddressUtils; import lbms.plugins.mldht.kad.utils.ByteWrapper; import lbms.plugins.mldht.kad.utils.PopulationEstimator; @@ -317,7 +309,9 @@ + r.getOrigin().getAddress()); // everything OK, so store the value - db.store(r.getInfoHash(), PeerAddressDBItem.createFromAddress(r.getOrigin().getAddress(), r.getPort(), r.isSeed())); + PeerAddressDBItem item = PeerAddressDBItem.createFromAddress(r.getOrigin().getAddress(), r.getPort(), r.isSeed()); + if(!AddressUtils.isBogon(item)) + db.store(r.getInfoHash(), item); // send a proper response to indicate everything is OK AnnounceResponse rsp = new AnnounceResponse(r.getMTID()); @@ -347,7 +341,7 @@ } InetSocketAddress addr = new InetSocketAddress(host, hport); - if (!addr.isUnresolved()) { + if (!addr.isUnresolved() && !AddressUtils.isBogon(addr)) { if(!type.PREFERRED_ADDRESS_TYPE.isInstance(addr.getAddress()) || node.getNumEntriesInRoutingTable() > DHTConstants.BOOTSTRAP_IF_LESS_THAN_X_PEERS) return; getRandomServer().ping(addr); Modified: mldht/branches/indexer/lbms/plugins/mldht/kad/Database.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/Database.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/Database.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -66,6 +66,7 @@ */ public void store(Key key, DBItem dbi) { + Set<DBItem> keyEntries = null; Set<DBItem> insertCanidate = new LightHashSet<DBItem>(); Modified: mldht/branches/indexer/lbms/plugins/mldht/kad/Node.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/Node.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/Node.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -30,6 +30,7 @@ import lbms.plugins.mldht.kad.tasks.PingRefreshTask; import lbms.plugins.mldht.kad.tasks.Task; import lbms.plugins.mldht.kad.tasks.TaskListener; +import lbms.plugins.mldht.kad.utils.AddressUtils; /** @@ -123,7 +124,7 @@ public void insertEntry (KBucketEntry entry, boolean internalInsert) { - if(usedIDs.containsValue(entry.getID())) + if(usedIDs.containsValue(entry.getID()) || AddressUtils.isBogon(entry.getAddress())) return; Key nodeID = entry.getID(); Modified: mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/NodeLookup.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/NodeLookup.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/NodeLookup.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -26,11 +26,10 @@ import lbms.plugins.mldht.kad.DHT.DHTtype; import lbms.plugins.mldht.kad.Key.DistanceOrder; import lbms.plugins.mldht.kad.Node.RoutingTableEntry; -import lbms.plugins.mldht.kad.messages.FindNodeRequest; -import lbms.plugins.mldht.kad.messages.FindNodeResponse; -import lbms.plugins.mldht.kad.messages.MessageBase; +import lbms.plugins.mldht.kad.messages.*; import lbms.plugins.mldht.kad.messages.MessageBase.Method; import lbms.plugins.mldht.kad.messages.MessageBase.Type; +import lbms.plugins.mldht.kad.utils.AddressUtils; import lbms.plugins.mldht.kad.utils.PackUtil; /** @@ -129,7 +128,7 @@ { // add node to todo list KBucketEntry e = PackUtil.UnpackBucketEntry(nodes, i * type.NODES_ENTRY_LENGTH, type); - if (!node.allLocalIDs().contains(e.getID()) && !todo.contains(e) && !visited.contains(e)) + if (!AddressUtils.isBogon(e.getAddress()) && !node.allLocalIDs().contains(e.getID()) && !todo.contains(e) && !visited.contains(e)) { todo.add(e); } Modified: mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/PeerLookupTask.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/PeerLookupTask.java 2010-08-23 23:04:43 UTC (rev 1930) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/tasks/PeerLookupTask.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -22,11 +22,9 @@ import lbms.plugins.mldht.kad.*; import lbms.plugins.mldht.kad.DHT.DHTtype; import lbms.plugins.mldht.kad.KBucketEntry.DistanceOrder; -import lbms.plugins.mldht.kad.messages.AnnounceRequest; -import lbms.plugins.mldht.kad.messages.GetPeersRequest; -import lbms.plugins.mldht.kad.messages.GetPeersResponse; -import lbms.plugins.mldht.kad.messages.MessageBase; +import lbms.plugins.mldht.kad.messages.*; import lbms.plugins.mldht.kad.messages.MessageBase.Method; +import lbms.plugins.mldht.kad.utils.AddressUtils; import lbms.plugins.mldht.kad.utils.PackUtil; /** @@ -120,7 +118,7 @@ { // add node to todo list KBucketEntry e = PackUtil.UnpackBucketEntry(nodes, i * type.NODES_ENTRY_LENGTH, type); - if(!node.allLocalIDs().contains(e.getID()) && !visited.contains(e)) + if(!AddressUtils.isBogon(e.getAddress()) && !node.allLocalIDs().contains(e.getID()) && !visited.contains(e)) todo.add(e); } } @@ -144,7 +142,8 @@ continue; PeerAddressDBItem it = (PeerAddressDBItem) item; // also add the items to the returned_items list - returnedItems.add(it); + if(!AddressUtils.isBogon(it)) + returnedItems.add(it); } KBucketEntry entry = new KBucketEntry(rsp.getOrigin(), rsp.getID()); Added: mldht/branches/indexer/lbms/plugins/mldht/kad/utils/AddressUtils.java =================================================================== --- mldht/branches/indexer/lbms/plugins/mldht/kad/utils/AddressUtils.java (rev 0) +++ mldht/branches/indexer/lbms/plugins/mldht/kad/utils/AddressUtils.java 2010-08-24 00:04:08 UTC (rev 1931) @@ -0,0 +1,30 @@ +package lbms.plugins.mldht.kad.utils; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import lbms.plugins.mldht.kad.PeerAddressDBItem; + +public class AddressUtils { + + public static boolean isBogon(PeerAddressDBItem item) + { + return isBogon(item.getInetAddress(), item.getPort()); + } + + public static boolean isBogon(InetSocketAddress addr) + { + return isBogon(addr.getAddress(),addr.getPort()); + } + + public static boolean isBogon(InetAddress addr, int port) + { + return port > 0 && port <= 0xFFFF && isGlobalUnicast(addr); + } + + public static boolean isGlobalUnicast(InetAddress addr) + { + return !(addr.isAnyLocalAddress() || addr.isLinkLocalAddress() || addr.isLoopbackAddress() || addr.isMulticastAddress() || addr.isSiteLocalAddress()); + } + +} Property changes on: mldht/branches/indexer/lbms/plugins/mldht/kad/utils/AddressUtils.java ___________________________________________________________________ Added: svn:mime-type + text/plain This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |