nasa-exp-checkins Mailing List for NASA WorldWind (Page 2)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 454-5900

Revision: 4081
          http://nasa-exp.svn.sourceforge.net/nasa-exp/?rev=4081&view=rev
Author:   ammianus
Date:     2009-08-23 13:14:26 +0000 (Sun, 23 Aug 2009)

Log Message:
-----------
1_4_1 Issue #3 - Fix for download code. 4xx / Client error are cached as failures and WW will not retry to download them during that session (cache is in memory only and cleared when WW is exited). 5xx / Server Error are recorded and a timeout period will begin before requests are allowed to this server again. A message is displayed in the top right of the UI stating the timeout length. If server response contains "retry-after" header, that period is used, otherwise default is 2 minutes. 

Modified Paths:
--------------
    trunk/WorldWind/PluginSDK/QuadTileSet.cs
    trunk/WorldWind/PluginSDK/Renderable/GeoSpatialDownloadRequest.cs
    trunk/WorldWind/WebDownload/WebDownload.cs

Modified: trunk/WorldWind/PluginSDK/QuadTileSet.cs
===================================================================

--- trunk/WorldWind/PluginSDK/QuadTileSet.cs	2009-08-03 02:02:31 UTC (rev 4080)
+++ trunk/WorldWind/PluginSDK/QuadTileSet.cs	2009-08-23 13:14:26 UTC (rev 4081)
@@ -65,6 +65,7 @@
 		protected GeoSpatialDownloadRequest[] m_activeDownloads = new GeoSpatialDownloadRequest[20];
 		protected DateTime[] m_downloadStarted = new DateTime[20];
 		protected TimeSpan m_connectionWaitTime = TimeSpan.FromMinutes(2);
+        protected TimeSpan m_defaultWaitTime = TimeSpan.FromMinutes(2);
 		protected DateTime m_connectionWaitStart;
 		protected bool m_isConnectionWaiting;
 		protected bool m_enableColorKeying;
@@ -75,8 +76,11 @@
 		protected string m_effectTechnique = null;
         protected Hashtable m_effectHandles = null;
 		static protected EffectPool m_effectPool = new EffectPool();
+        protected Hashtable m_failedDownloads = new Hashtable();
+        //this value should be >= 1
+        private int MaxRetriesPerTile = 1;
+        private int MaxFailedTilesForTimeout = 5;
 
-
 		protected TimeSpan m_cacheExpirationTime = TimeSpan.MaxValue;
 
 
@@ -181,7 +185,7 @@
 
 			// Default terrain mapped imagery to terrain mapped priority
 			if (terrainMapped)
-				RenderPriority = RenderPriority.TerrainMappedImages;
+                RenderPriority = RenderPriority.TerrainMappedImages;
 		}
 
 		/// <summary>
@@ -921,9 +925,21 @@
 			{
 				if (IsConnectionWaiting)
 				{
-					if (DateTime.Now.Subtract(TimeSpan.FromSeconds(15)) < ConnectionWaitStart)
+					if (DateTime.Now.Subtract(TimeSpan.FromSeconds(20)) < ConnectionWaitStart)
 					{
-						string s = "Problem connecting to server... Trying again in 2 minutes.\n";
+                        //the wait time may vary, display the correct wait time in minutes and seconds
+                        int minutesWaiting = m_connectionWaitTime.Minutes;
+                        String timePart;
+                        if (minutesWaiting > 0)
+                        {
+                            timePart = m_connectionWaitTime.Minutes + " minutes "+m_connectionWaitTime.Seconds+" seconds";
+                        }
+                        else
+                        {
+                            timePart = m_connectionWaitTime.Seconds + " seconds";
+                        }
+                        //this message is rendered in upper right corner
+						string s = "Problem connecting to server... Trying again in "+timePart+".\n";
 						drawArgs.UpperLeftCornerText += s;
 					}
 				}
@@ -1086,6 +1102,12 @@
                 }
             }
 
+            lock (m_failedDownloads.SyncRoot)
+            {
+                //clear this hashtable                
+                m_failedDownloads.Clear();
+            }
+
 			foreach (QuadTile qt in m_topmostTiles.Values)
 				qt.Dispose();
 
@@ -1124,6 +1146,14 @@
 				foreach (long deletionKey in deletionList)
 					m_topmostTiles.Remove(deletionKey);
 			}
+
+            //reset the numberRetries and timeout, also reset failedDownloads Hashtable
+            m_isConnectionWaiting = false;
+            //m_connectionWaitStart = System.DateTime.MinValue;
+            m_failedDownloads.Clear();
+            m_numberRetries = 0;
+            Log.Write(Log.Levels.Verbose, "QTS", " ResetCacheForCurrentView: (" + m_isConnectionWaiting + "," + m_connectionWaitStart + "," +m_failedDownloads.Count +"," +m_numberRetries);
+                        
 		}
 
 		public void ClearDownloadRequests()
@@ -1143,37 +1173,52 @@
                 if (m_downloadRequests.Contains(key))
 					return;
 
-                Log.Write(Log.Levels.Verbose, "QTS", "AddToDownloadQueue: " + newRequest.QuadTile.ToString());
+                //go through download requests, remove any requests that have failed before
+                if (IsTileFailed(newRequest.QuadTile.Level,newRequest.QuadTile.Col,newRequest.QuadTile.Row))
+                {
+                    
+                    //increment number of retries overall for this tile (more than 5 failed downloads and anti-hammer goes into effect for wait time)
+                    Log.Write(Log.Levels.Verbose, "QTS", "AddToDownloadQueue: Requested tile will not download again.");
+                    newRequest.QuadTile.WaitingForDownload = false;
+                    
+                }
+                
 
-				m_downloadRequests.Add(key, newRequest);
+                if(newRequest.QuadTile.WaitingForDownload == true)
+                {
 
-				if (m_downloadRequests.Count >= m_maxQueueSize)
-				{
-					//remove spatially farthest request
-					GeoSpatialDownloadRequest farthestRequest = null;
-					Angle curDistance = Angle.Zero;
-					Angle farthestDistance = Angle.Zero;
-					foreach (GeoSpatialDownloadRequest curRequest in m_downloadRequests.Values)
-					{
-						curDistance = MathEngine.SphericalDistance(
-										curRequest.QuadTile.CenterLatitude,
-										curRequest.QuadTile.CenterLongitude,
-										camera.Latitude,
-										camera.Longitude);
+                    Log.Write(Log.Levels.Verbose, "QTS", "AddToDownloadQueue: " + newRequest.QuadTile.ToString());
 
-						if (curDistance > farthestDistance)
-						{
-							farthestRequest = curRequest;
-							farthestDistance = curDistance;
-						}
-					}
+				    m_downloadRequests.Add(key, newRequest);
 
-					farthestRequest.Dispose();
-					farthestRequest.QuadTile.DownloadRequests.Remove(farthestRequest);
-					m_downloadRequests.Remove(farthestRequest.QuadTile);
-				}
-			}
+				    if (m_downloadRequests.Count >= m_maxQueueSize)
+				    {
+					    //remove spatially farthest request
+					    GeoSpatialDownloadRequest farthestRequest = null;
+					    Angle curDistance = Angle.Zero;
+					    Angle farthestDistance = Angle.Zero;
+					    foreach (GeoSpatialDownloadRequest curRequest in m_downloadRequests.Values)
+					    {
+						    curDistance = MathEngine.SphericalDistance(
+										    curRequest.QuadTile.CenterLatitude,
+										    curRequest.QuadTile.CenterLongitude,
+										    camera.Latitude,
+										    camera.Longitude);
 
+						    if (curDistance > farthestDistance)
+						    {
+							    farthestRequest = curRequest;
+							    farthestDistance = curDistance;
+						    }
+					    }
+
+					    farthestRequest.Dispose();
+					    farthestRequest.QuadTile.DownloadRequests.Remove(farthestRequest);
+					    m_downloadRequests.Remove(farthestRequest.QuadTile);
+				    }
+			    }
+            }
+
 			ServiceDownloadQueue();
 		}
 
@@ -1223,22 +1268,27 @@
 					m_activeDownloads[i].Dispose();
 					m_activeDownloads[i] = null;
 				}
-
-				if (NumberRetries >= 5 || m_isConnectionWaiting)
+                
+                
+                //If there have been 'MaxFailedTilesForTimeout' or currently in a Timeout
+                if (NumberRetries >= MaxFailedTilesForTimeout || m_isConnectionWaiting)
 				{
-					// Anti hammer in effect
+                    
+                    // Anti hammer in effect
 					if (!m_isConnectionWaiting)
 					{
-						m_connectionWaitStart = DateTime.Now;
-						m_isConnectionWaiting = true;
+                        Log.Write(Log.Levels.Debug, "QTS", "ServiceDownloadQueue: Anti-Hammer[" + NumberRetries + "," + m_isConnectionWaiting + "," + m_connectionWaitStart.ToShortTimeString() + "] ");
+                        //use default timeout (2 minutes)
+                        setTimeoutAndWait(m_defaultWaitTime);
 					}
 
-					if (DateTime.Now.Subtract(m_connectionWaitTime) > m_connectionWaitStart)
+					//after waiting for wait time clear the number of retries
+                    if (DateTime.Now.Subtract(m_connectionWaitTime) > m_connectionWaitStart)
 					{
 						NumberRetries = 0;
 						m_isConnectionWaiting = false;
 					}
-                    m_downloadRequests.Clear();
+                    ClearDownloadRequests();
 					return;
 				}
 
@@ -1343,5 +1393,67 @@
 				return texture;
 			}
 		}
+
+        /// <summary>
+        /// When a DownloadRequest fails when a server returns a 404 error this will record that failure for the QuadTile
+        /// If the QuadTile fails to be downloaded after 'MaxRetriesPerTile' retries the QuadTileSet will not download it again to prevent
+        /// hammering the server.
+        /// </summary>
+        /// <param name="downloadRequest"></param>
+        public void RecordFailedRequest(GeoSpatialDownloadRequest downloadRequest)
+        {
+            //use the QuadTile's String as a key, assuming this would be unique
+            String quadTileKey = downloadRequest.QuadTile.ToString();
+            QuadTile qt = downloadRequest.QuadTile;
+            
+            int numberFailures = 1;
+            if (m_failedDownloads.ContainsKey(quadTileKey))
+            {
+                //already has failed once, increment the number of failures
+                numberFailures = (int) m_failedDownloads[quadTileKey];
+                numberFailures++;
+                m_failedDownloads[quadTileKey] =  numberFailures;
+                
+            }else{
+                //first time failed, add to the hashTable
+                m_failedDownloads.Add(quadTileKey, numberFailures);
+            }
+        }
+
+        /// <summary>
+        /// Allows for setting the Timeout for this QuadTileSet for a specified length of time
+        /// where downloads will be put on hold.
+        /// </summary>
+        /// <param name="waitTime">How long to set wait</param>
+        public void setTimeoutAndWait(TimeSpan waitTime)
+        {
+            m_connectionWaitStart = DateTime.Now;
+            m_isConnectionWaiting = true;
+            m_connectionWaitTime = waitTime;
+        }
+
+        /// <summary>
+        /// Checks if the requested download has been flagged as a failed download
+        /// </summary>
+        /// <param name="level"></param>
+        /// <param name="x"></param>
+        /// <param name="y"></param>
+        /// <returns></returns>
+        public Boolean IsTileFailed(int level, int x, int y)
+        {
+            Boolean isTileFailed = false;
+            //construct key
+            String key = String.Format("QuadTile:Set={0} Level={1} X={2} Y={3}",
+                this.Name, level, x, y);
+            //TODO calculate tiles below failed tile as well. ie need to look up at lower levels for failures
+            if(m_failedDownloads.ContainsKey(key))
+            {
+                isTileFailed = true;
+            }
+            return isTileFailed;
+        }
+
 	}
+
+    
 }

Modified: trunk/WorldWind/PluginSDK/Renderable/GeoSpatialDownloadRequest.cs
===================================================================
--- trunk/WorldWind/PluginSDK/Renderable/GeoSpatialDownloadRequest.cs	2009-08-03 02:02:31 UTC (rev 4080)
+++ trunk/WorldWind/PluginSDK/Renderable/GeoSpatialDownloadRequest.cs	2009-08-23 13:14:26 UTC (rev 4081)
@@ -75,7 +75,7 @@
 			{
 				downloadInfo.Verify();
 
-				m_quadTile.QuadTileSet.NumberRetries = 0;
+				//m_quadTile.QuadTileSet.NumberRetries = 0;
 
 				// Rename temp file to real name
 				File.Delete(m_localFilePath);
@@ -88,13 +88,69 @@
 			catch(System.Net.WebException caught)
 			{
 				System.Net.HttpWebResponse response = caught.Response as System.Net.HttpWebResponse;
-				if(response!=null && response.StatusCode==System.Net.HttpStatusCode.NotFound)
+                /*
+                 * null response
+                 */
+                if (response == null)
+                {
+                    m_quadTile.QuadTileSet.RecordFailedRequest(this);
+                }
+                /* 4xx - Client error
+                 * 400 Bad Request
+                 * 401 Unauthorized
+                 * 403 Forbidden
+                 * 404 Not Found
+                 * 206 Partial Content
+                 * 200 OK && Content length == 0
+                 */
+                else if(response.StatusCode==System.Net.HttpStatusCode.NotFound || 
+                    response.StatusCode==System.Net.HttpStatusCode.Unauthorized ||
+                    response.StatusCode==System.Net.HttpStatusCode.Forbidden ||
+                    response.StatusCode==System.Net.HttpStatusCode.BadRequest ||
+                    response.StatusCode == System.Net.HttpStatusCode.PartialContent ||
+                    (response.StatusCode == System.Net.HttpStatusCode.OK && 
+                        response.ContentLength == 0))
 				{
-					using(File.Create(m_localFilePath + ".txt"))
-					{}
-					return;
+                    m_quadTile.QuadTileSet.RecordFailedRequest(this);
+                    
 				}
-				m_quadTile.QuadTileSet.NumberRetries++;
+                /* 
+                 * 5xx - Server Error
+                 * 500 Internal Server Error
+                 * 501 Not Implemented
+                 * 502 Bad Gateway
+                 * 503 Service Unavailable
+                 */
+                else if (response.StatusCode == System.Net.HttpStatusCode.InternalServerError ||
+                        response.StatusCode == System.Net.HttpStatusCode.NotImplemented ||
+                        response.StatusCode == System.Net.HttpStatusCode.BadGateway ||
+                        response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable
+                        )
+                {
+                    //server problem, directly start timeout for the layer, rather than counter per tile
+
+
+                    TimeSpan waitTime = TimeSpan.FromSeconds(120);
+                    //if retry-after is specified then wait for that length of time before retrying
+                    String retryAfter = response.GetResponseHeader("Retry-After");
+                    if (retryAfter != null && !retryAfter.Equals(String.Empty))
+                    {
+                        Log.Write(Log.Levels.Verbose, "GSDR", "Retry-After response header " + retryAfter);
+                        try
+                        {
+                            //try to convert
+                            double retryAfterNumber = Convert.ToDouble(retryAfter);
+                            waitTime = TimeSpan.FromSeconds(retryAfterNumber);
+                        }
+                        catch (System.FormatException fe)
+                        {
+                            //ignore retry-after, just wait for 60 seconds
+                        }
+                    }
+                    //wait before retrying
+                    m_quadTile.QuadTileSet.setTimeoutAndWait(waitTime);
+                }
+                
 			}
 			catch
 			{

Modified: trunk/WorldWind/WebDownload/WebDownload.cs
===================================================================
--- trunk/WorldWind/WebDownload/WebDownload.cs	2009-08-03 02:02:31 UTC (rev 4080)
+++ trunk/WorldWind/WebDownload/WebDownload.cs	2009-08-23 13:14:26 UTC (rev 4081)
@@ -1024,15 +1024,21 @@
             if (Exception != null)
             {
                 //these occur regularly - ignore
-                if (Exception is WebException)
-                {
-                    Utility.Log.Write(Log.Levels.Debug, "NET", "Verify(): WebException: " + Exception.Message);
-                }
-                else
-                {
-                    throw Exception;
-                }
+                //no - let these errors also rise up to the request to handle (ie a 404 will not get requested again)
+                throw Exception;
             }
+            else if (response.StatusCode == System.Net.HttpStatusCode.PartialContent)
+            {
+                //treat 206 Partial Content as an Exception
+                WebException partialDownloadException = new WebException("Partial Download Response",null,WebExceptionStatus.ReceiveFailure,response);
+                throw partialDownloadException;
+            }
+            else if (response.StatusCode == System.Net.HttpStatusCode.OK && response.ContentLength == 0)
+            {
+                //treat 200 OK and content length == 0 as an Exception
+                WebException noContentException = new WebException("No Content OK Download Response", null, WebExceptionStatus.ReceiveFailure, response);
+                throw noContentException;
+            }
 		}
 
 		/// <summary>


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




2005	Jan	Feb	Mar	Apr	May (160)	Jun (302)	Jul (417)	Aug (96)	Sep (42)	Oct (135)	Nov (223)	Dec (22)
2006	Jan (36)	Feb (29)	Mar (70)	Apr (94)	May (96)	Jun (57)	Jul (97)	Aug (178)	Sep (235)	Oct (225)	Nov (47)	Dec (57)
2007	Jan (111)	Feb (41)	Mar (66)	Apr (38)	May (51)	Jun (7)	Jul (49)	Aug (6)	Sep (33)	Oct (17)	Nov (10)	Dec (15)
2008	Jan (16)	Feb (8)	Mar (11)	Apr	May (1)	Jun (10)	Jul (17)	Aug	Sep	Oct	Nov	Dec (2)
2009	Jan (2)	Feb (23)	Mar (12)	Apr	May	Jun	Jul (2)	Aug (7)	Sep (1)	Oct	Nov (1)	Dec (3)
2010	Jan	Feb (1)	Mar (2)	Apr	May	Jun	Jul	Aug (2)	Sep (6)	Oct	Nov	Dec
2011	Jan	Feb	Mar	Apr	May (1)	Jun (1)	Jul	Aug	Sep	Oct (1)	Nov	Dec
2012	Jan	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct (3)	Nov	Dec
2014	Jan	Feb	Mar	Apr	May (2)	Jun	Jul	Aug	Sep	Oct	Nov	Dec

nasa-exp-checkins Mailing List for NASA WorldWind (Page 2)

nasa-exp-checkins — CVS commit messages