[Mapproxy-java] SF.net SVN: mapproxy:[39] core/cache
Brought to you by:
pborissow
Menu
▾
▴
[Mapproxy-java] SF.net SVN: mapproxy:[39] core/cache
|
From: <pbo...@us...> - 2012-01-10 11:43:52
|
Revision: 39
http://mapproxy.svn.sourceforge.net/mapproxy/?rev=39&view=rev
Author: pborissow
Date: 2012-01-10 11:43:40 +0000 (Tue, 10 Jan 2012)
Log Message:
-----------
- Sync'd baseline with changes from GeoEye dated Oct 6. Changes are focused on improvements to the seeding capability.
- Deprecated the "mapproxy.server" package. Functionality is replaced with the javaxt-server library and the new HttpServlet class.
- Added unique IDs to Layers (required for seeding).
Modified Paths:
--------------
Main.java
config/Config.java
core/Layer.java
core/ServiceResponse.java
core/Status.java
core/TileSeeder.java
core/cache/Cache.java
core/cache/_Tile.java
Added Paths:
-----------
core/HttpServlet.java
core/InvalidTileException.java
core/TileConsumer.java
core/TileListener.java
Modified: Main.java
===================================================================
--- Main.java 2012-01-10 11:42:33 UTC (rev 38)
+++ Main.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -6,11 +6,7 @@
import mapproxy.core.*;
import mapproxy.wms.cache.WMSTileSource;
import mapproxy.config.Config;
-import mapproxy.server.HttpServer;
-import java.util.concurrent.ConcurrentHashMap;
-
-
//******************************************************************************
//** Main Application
//******************************************************************************
@@ -61,7 +57,6 @@
//testCache(url);
//testSRS();
- args = new String[]{"C:\\Documents and Settings\\peter.borissow\\My Documents\\Java\\Libraries\\mapproxy\\src\\mapproxy\\config.xml"};
startServer(args);
//seedTest(args);
//testWebServices();
@@ -252,11 +247,6 @@
String url = "http://services.arcgisonline.com/ArcGIS/services/USA_Topo_Maps/MapServer/?wsdl";
url = "http://sampleserver1.arcgisonline.com/arcgis/services/Specialty/ESRI_StateCityHighway_USA/MapServer?wsdl";
javaxt.webservices.WSDL wsdl = new javaxt.webservices.WSDL(new javaxt.http.Request(url).getResponse().getXML());
- javaxt.webservices.Soap soap = new javaxt.webservices.Soap(wsdl);
- soap.setHeader("Accept", "*/*");
- soap.setHeader("Content-Type", "text/xml");
- soap.setHeader("User-Agent", "ArcGIS Client Using WinInet");
- //soap.setHeader("SOAPAction", "\"\"");
//Flow: GetMessageVersion, GetFolders, GetServiceDescriptions
@@ -281,8 +271,15 @@
parameters.setValue("MapName", "Layers");
- new javaxt.io.File("/temp/GetLegendInfo.xml").write(soap.execute(service, method, parameters).getText(), "UTF-8");
+
+ javaxt.webservices.SoapRequest request = new javaxt.webservices.SoapRequest(service, method, parameters);
+ request.setHeader("Accept", "*/*");
+ request.setHeader("Content-Type", "text/xml");
+ request.setHeader("User-Agent", "ArcGIS Client Using WinInet");
+ //request.setHeader("SOAPAction", "\"\"");
+ new javaxt.io.File("/temp/GetLegendInfo.xml").write(request.getResponse().toString(), "UTF-8");
+
}
@@ -303,8 +300,17 @@
int numThreads = Config.base_config().get("threads").toInteger();
int port = Config.base_config().get("port").toInteger();
- HttpServer mapserver = new HttpServer(port, numThreads);
- mapserver.start();
+ //Start the server
+ try {
+ javaxt.http.Server mapserver = new javaxt.http.Server(port, numThreads, new HttpServlet());
+ mapserver.start();
+ }
+ catch (Exception e) {
+ System.out.println("Server could not start because of an "
+ + e.getClass());
+ System.out.println(e);
+ return;
+ }
}
@@ -313,13 +319,18 @@
double[] bbox = new double[]{-180,-90,180,90};
String srs = "EPSG:4326";
- int startLevel = 1;
- int endLevel = 12;
+ int startLevel = 6;
+ int endLevel = 6;
Config.load(getConfig(args));
Service service = ((mapproxy.core.Service) mapproxy.config.Config.base_config().get("mapproxy").toObject());
+ System.out.println("Num Layers: " + service.getLayers().length);
Layer layer = service.getLayers()[0];
- Status status = layer.seed(2, bbox, srs, startLevel, endLevel);
+ System.out.println("Layer Name: " + layer.getName());
+ System.out.println("Layer ID: " + layer.getID());
+ //if (true) return;
+ //layer.setParam("id", 0);
+ Status status = layer.seed(2, bbox, srs, startLevel, endLevel, null);
long lastUpdate = -1;
@@ -360,6 +371,7 @@
private org.w3c.dom.Document getConfig(String[] args) throws Exception {
if (args[args.length-1].toLowerCase().endsWith(".xml")){
javaxt.io.File file = new javaxt.io.File(args[args.length-1]);
+
if (!file.exists()) throw new Exception("Config file not found: " + file);
else {
return file.getXML();
@@ -407,7 +419,12 @@
CacheManager cacheManager = new CacheManager(fileCache, wms);
Cache cache = new Cache(cacheManager, grid, true);
- cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size()).saveAs("/temp/ew_cache.png");
+ try{
+ cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size()).saveAs("/temp/ew_cache.png");
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ }
}
Modified: config/Config.java
===================================================================
--- config/Config.java 2012-01-10 11:42:33 UTC (rev 38)
+++ config/Config.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -16,7 +16,10 @@
public static Options options;
public static Options base_config(){
- if (options==null) options = new Options();
+ if (options==null){
+ options = new Options();
+ options.set("version", "1.0");
+ }
return options;
}
@@ -203,7 +206,8 @@
if (service!=null){
java.util.ArrayList<mapproxy.core.Layer> arr = new java.util.ArrayList<mapproxy.core.Layer>();
for (String str : layers.split(",")){
- arr.add(service.getLayer(str));
+ mapproxy.core.Layer lyr = service.getLayer(str);
+ if (lyr!=null) arr.add(service.getLayer(str));
}
mapproxy.core.BBox bbox = service.getBBox(arr.toArray(new mapproxy.core.Layer[arr.size()]));
layer.setBBox(bbox);
Added: core/HttpServlet.java
===================================================================
--- core/HttpServlet.java (rev 0)
+++ core/HttpServlet.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -0,0 +1,171 @@
+package mapproxy.core;
+import javaxt.http.servlet.*;
+import java.util.concurrent.ConcurrentHashMap;
+
+//******************************************************************************
+//** HttpServlet Class
+//******************************************************************************
+/**
+ * Implementation of an HttpServlet used to process HTTP requests and
+ * generate a response.
+ *
+ ******************************************************************************/
+
+public class HttpServlet implements javaxt.http.servlet.HttpServlet {
+
+ private static ConcurrentHashMap<String, String> httpCache;
+ private static String serverName;
+
+ //**************************************************************************
+ //** Constructor
+ //**************************************************************************
+ /** Creates a new instance of HttpServlet. */
+
+ public HttpServlet() {
+ serverName = "MapProxy Server " + mapproxy.config.Config.base_config().get("version").toString();
+ httpCache = new ConcurrentHashMap<String, String>();
+ }
+
+
+ //**************************************************************************
+ //** processRequest
+ //**************************************************************************
+ /** Used to process http get and post requests. */
+
+ public void processRequest(HttpServletRequest request, HttpServletResponse response) {
+ //long startTime = new java.util.Date().getTime();
+ //long initResponse = 0;
+ //long initCache = 0;
+ try{
+ Service service = ((mapproxy.core.Service) mapproxy.config.Config.base_config().get("mapproxy").toObject());
+ ServiceResponse map = service.getResponse(request.getURL().toString(), new String(request.getBody(), "UTF-8"));
+ javaxt.utils.Date date = map.getDate();
+ String key = map.getID();
+ //initResponse = new java.util.Date().getTime()-startTime;
+ boolean useCache = useCache(request, key);
+ //initCache = new java.util.Date().getTime()-(initResponse+startTime);
+ if (useCache){
+
+ //Return 304 Response Code
+ response.setStatus(304, "Not Modified");
+ response.setHeader("Date", date.toString("EEE, dd MMM yyyy HH:mm:ss zzz"));
+ response.setHeader("Server", serverName);
+
+ }
+ else{
+
+ byte[] rsp = map.getByteArray();
+ String contentType = map.getContentType();
+ String eTag = "W/\"" + rsp.length + "-" + date.getTime() + "\"";
+
+
+ //Set header
+ response.setHeader("Date", date.toString("EEE, dd MMM yyyy HH:mm:ss zzz"));
+ response.setHeader("Server", serverName);
+ response.setHeader("Content-Type", contentType);
+ response.setHeader("Content-Length", rsp.length+"");
+ response.setHeader("ETag", eTag);
+ response.setHeader("Last-Modified", date.toString("EEE, dd MMM yyyy HH:mm:ss zzz")); //Sat, 23 Oct 2010 13:04:28 GMT
+ //header.append("Cache-Control", "max-age=" + (365*24*60*60)); //vs "no-cache"
+ //header.append("Accept-Ranges", "bytes");
+
+
+ //Write body
+ response.write(rsp);
+
+
+ //Update the http cache
+ this.updateCache(key, eTag);
+ }
+ }
+ catch(Exception e){
+
+ //Return Error
+ response.setStatus(500);
+ response.setHeader("Server", serverName);
+ response.write(e.getLocalizedMessage());
+ }
+
+ //long ttl = new java.util.Date().getTime()-startTime;
+ //System.out.println(ttl + "\t" + (double)initResponse/(double)ttl + "\t" + (double)initCache/(double)ttl);
+ }
+
+
+
+ //**************************************************************************
+ //** useCache
+ //**************************************************************************
+ /** Used to check whether the request is in the http cache. If so, sends a
+ * 304 http response and returns true. Otherwise, nothing is sent back to
+ * the client and the method returns false.
+ */
+ private boolean useCache(HttpServletRequest request, String key) {
+
+ if (key==null) return false;
+
+
+ String cacheControl = request.getHeader("cache-control");
+ if (cacheControl==null) cacheControl = "";
+ if (cacheControl.equalsIgnoreCase("no-cache")==false){
+
+ //Find the eTag that corresponds to this request
+ String eTag = null;
+ synchronized (httpCache){
+ eTag = httpCache.get(key);
+ }
+
+ if (eTag!=null){
+
+ String matchTag = request.getHeader("if-none-match");
+ if (matchTag==null) matchTag = "";
+ if (matchTag.equals(eTag)){
+ //System.out.println("if-none-match");
+ return true;
+ }
+ else{
+ //Internet Explorer 6 uses "if-modified-since" instead of "if-none-match"
+ matchTag = request.getHeader("if-modified-since");
+ if (matchTag!=null){
+ String time = eTag.substring(eTag.lastIndexOf("-")+1, eTag.length()-1);
+ try{
+
+ javaxt.utils.Date date = new javaxt.utils.Date(Long.parseLong(time));
+ date.setTimeZone("GMT");
+ String d = date.toString("EEE, dd MMM yyyy HH:mm:ss zzz");
+
+ for (String tag: matchTag.split(";")){
+ if (tag.trim().equals(d)){
+ //System.out.println("if-modified-since");
+ return true;
+ }
+ }
+ }
+ catch(Exception e){
+ }
+
+ }
+
+ }
+
+ }
+
+ }
+
+
+ return false;
+ }
+
+
+ //**************************************************************************
+ //** updateCache
+ //**************************************************************************
+ /** Used to update the http cache.
+ */
+ private void updateCache(String key, String eTag){
+ if (key==null || eTag==null) return;
+ synchronized (httpCache){
+ httpCache.put(key, eTag); //<-- Need to make this smarter so we don't run out of memory...
+ }
+ }
+
+}
\ No newline at end of file
Added: core/InvalidTileException.java
===================================================================
--- core/InvalidTileException.java (rev 0)
+++ core/InvalidTileException.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -0,0 +1,24 @@
+package mapproxy.core;
+
+//******************************************************************************
+//** Service Exception
+//******************************************************************************
+/**
+ * Used to represent a server exception. Not yet implemented or used.
+ *
+ * Note that this class was not part of the original mapproxy baseline.
+ *
+ ******************************************************************************/
+
+public class InvalidTileException extends Exception {
+
+
+ //**************************************************************************
+ //** Constructor
+ //**************************************************************************
+ /** Creates a new instance of Exception. */
+
+ public InvalidTileException() {
+
+ }
+}
\ No newline at end of file
Modified: core/Layer.java
===================================================================
--- core/Layer.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/Layer.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -6,7 +6,11 @@
import mapproxy.wms.cache.WMSTileSource;
import mapproxy.arcgis.cache.*;
import mapproxy.arcgis.client.*;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import javaxt.io.Directory;
+import javaxt.io.File;
//******************************************************************************
//** Layer Class
@@ -25,9 +29,19 @@
private BBox bbox;
private java.util.HashSet<String> srs = new java.util.HashSet<String>();
private java.util.List<Layer> layers = new java.util.ArrayList<Layer>();
+ private static java.util.List<String> layerIDs = new java.util.ArrayList<String>();
+
private int parentID = -1;
private java.util.HashMap<String, Object> params = new java.util.HashMap<String, Object>();
+ private static final ConcurrentHashMap<String, TileSeeder> ACTIVE_TASKS = new ConcurrentHashMap<String, TileSeeder>();
+ // blank images to return if bbox does not intersect any catalog items
+ private static javaxt.io.Image image512; // 512 x 512
+ private static javaxt.io.Image image256; // 256 x 256
+ static {
+ image512 = new javaxt.io.Image(512, 512);
+ image256 = new javaxt.io.Image(256, 256);
+ }
//**************************************************************************
//** Constructor
@@ -35,16 +49,38 @@
/** Creates a new instance of Layer. */
public Layer(String name) {
- this.name = name;
+ this(name, -1);
}
public Layer(String name, int parentID) {
this.name = name;
this.parentID = parentID;
+
+ String id = generateKey();
+ synchronized(layerIDs){
+ while (layerIDs.contains(id)){
+ id = generateKey();
+ }
+ layerIDs.add(id);
+ layerIDs.notifyAll();
+ }
+ params.put("id", id);
}
+
+
//**************************************************************************
+ //** getID
+ //**************************************************************************
+ /** Returns the layer ID.
+ */
+ public String getID(){
+ return params.get("id").toString();
+ }
+
+
+ //**************************************************************************
//** getParentID
//**************************************************************************
/** Returns the parent layer ID. The parent ID should be a positive integer.
@@ -218,16 +254,32 @@
}
- @Override
+ //@Override
public int compareTo(Object obj){
if (obj==null) return -1;
else return -obj.toString().compareTo(name.toUpperCase());
}
+ public void invalidateTile(int[] tile, String format, String srs) {
+ FileCache fileCache = (FileCache) this.getParam("cache");
-
-
-
+ //Check whether we should use/create a map cache
+ boolean useCache = (fileCache!=null);
+ if (useCache) {
+ _Tile _tile = new _Tile(tile);
+ String location = fileCache.tile_location(_tile, false);
+ if (location != null && location.length() != 0) {
+ File file = new File(location);
+ if (file.exists()) {
+ file.delete();
+ // System.err.println("invalid png deleted" + location);
+ } else {
+ // System.err.println("invalid png" + location);
+ }
+ }
+ }
+ }
+
//**************************************************************************
//** getImage
//**************************************************************************
@@ -258,8 +310,20 @@
//Check whether we should use/create a map cache
boolean useCache = (fileCache!=null);
- if (useCache){
- useCache = this.getBBox().intersects(new BBox(bbox, srs));
+ if (useCache) {
+ if (!this.getBBox().intersects(new BBox(bbox, srs))) {
+ // return the blank tile, no need to call the WMS
+ javaxt.io.Image image = null;
+
+ if (width == 512 && height == 512) {
+ image = image512;
+ } else if (width == 256 && height == 256) {
+ image = image256;
+ } else {
+ image = new javaxt.io.Image(width, height);
+ }
+ return image;
+ }
}
@@ -282,7 +346,14 @@
WMSTileSource wms = new WMSTileSource(grid, new WMSClient(req));
CacheManager cacheManager = new CacheManager(fileCache, wms);
Cache cache = new Cache(cacheManager, grid, true);
- return cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size());
+ javaxt.io.Image image = null;
+ try {
+ image = cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size());
+ }
+ catch (Exception e) {
+
+ }
+ return image;
}
}
else if (protocol.equalsIgnoreCase("MapServer")){
@@ -294,7 +365,14 @@
MapTileSource arcMapServer = new MapTileSource(grid, new MapClient(req));
CacheManager cacheManager = new CacheManager(fileCache, arcMapServer);
Cache cache = new Cache(cacheManager, grid, true);
- return cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size());
+ javaxt.io.Image image = null;
+ try {
+ image = cache.image(req.params.bbox(), new SRS(req.params.srs()), req.params.size());
+ }
+ catch (Exception e) {
+
+ }
+ return image;
}
@@ -322,8 +400,18 @@
return this.getImage(width, height, format, bbox[0] + "," + bbox[1] + "," + bbox[2] + "," + bbox[3], srs);
}
+ public Status seed(int numThreads, double[] bbox, String srs, int startLevel, int endLevel,
+ TileConsumer tileProcessor) {
+ TileSeeder tileseeder = new TileSeeder();
+ synchronized (ACTIVE_TASKS) {
+ ACTIVE_TASKS.put(getParam("id").toString(), tileseeder);
+ ACTIVE_TASKS.notifyAll();
+ }
+ return tileseeder.execute(this, numThreads, bbox, startLevel, endLevel, false, tileProcessor);
+ }
+
//**************************************************************************
//** seedLayers
//**************************************************************************
@@ -336,13 +424,73 @@
* "Total Files", "Total Processed", and "Is Complete".
*/
public Status seed(int numThreads, double[] bbox, String srs, int startLevel, int endLevel){
- if (numThreads<1) numThreads = 1;
- for (int i=0; i<numThreads; i++){
- Thread t = new Thread(new TileSeeder());
- t.start();
+ return seed(numThreads, bbox, srs, startLevel, endLevel, null);
+ }
+
+ public boolean stopSeedCache() {
+ boolean success= true;
+ synchronized (ACTIVE_TASKS) {
+ TileSeeder tileSeeder = ACTIVE_TASKS.get(getParam("id").toString());
+ if (tileSeeder != null) {
+ success = tileSeeder.stop();
+ }
+ ACTIVE_TASKS.remove(getParam("id").toString());
+ ACTIVE_TASKS.notifyAll();
}
+ return success;
+ }
+
+ public void invalidate(int numThreads, double[] bbox, String srs, int startLevel, int endLevel, TileConsumer tileProcessor){
+ new TileSeeder().execute(this, numThreads, bbox, -1, -1, true, tileProcessor);
+ }
- return TileSeeder.addLayer(this, bbox, startLevel, endLevel);
+ //returns the levels cached so far by looking at the cache directory
+ public List<Integer> getCachedLevels() {
+ List<Integer> levels = new ArrayList<Integer>();
+ FileCache filecache = (FileCache)getParam("cache");
+ if (filecache != null) {
+ javaxt.io.Directory dir = new javaxt.io.Directory(filecache.cache_dir);
+ if (dir.exists()) {
+ Directory[] children = dir.getSubDirectories();
+ if (children != null && children.length != 0) {
+ for (int i=0; i < children.length; i++) {
+ Directory subDir = children[i];
+ try {
+ int n = Integer.parseInt(subDir.getName());
+ levels.add(n);
+ }
+ catch(NumberFormatException n) {
+
+ }
+ }
+ }
+ }
+ }
+ return levels;
}
+
+ //**************************************************************************
+ //** generateKey
+ //**************************************************************************
+ /** Used to generate a 36 character unique id */
+
+ private String generateKey(){
+ return CreateGUID(8) + "-" +
+ CreateGUID(4) + "-" +
+ CreateGUID(4) + "-" +
+ CreateGUID(4) + "-" +
+ CreateGUID(12);
+ }
+
+
+ private String CreateGUID(int tmpLength){
+ String tmpGUID = "";
+ final String strValid = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ for (int i = 1; i<=tmpLength; i++){
+ int x = new java.util.Random().nextInt(strValid.length());
+ tmpGUID += strValid.substring(x,x+1);
+ }
+ return tmpGUID;
+ }
}
\ No newline at end of file
Modified: core/ServiceResponse.java
===================================================================
--- core/ServiceResponse.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/ServiceResponse.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -88,7 +88,7 @@
* applications where a server may only need the key to implement http
* cache control.
*/
- private void init(boolean getBytes){
+ private void init(boolean getBytes) throws InvalidTileException {
if (getBytes && this.rsp!=null) return;
@@ -239,7 +239,7 @@
/** Returns a byte array containing the response. The byte array can be used
* in the body of an http response.
*/
- public byte[] getByteArray(){
+ public byte[] getByteArray() throws Exception{
this.init(true); //<-- Inititalize the byte array...
return rsp;
}
@@ -308,7 +308,7 @@
//**************************************************************************
/** Used to construct a response to a TMS Map request
*/
- private void getTile(boolean getBytes) {
+ private void getTile(boolean getBytes) throws InvalidTileException{
//Set local variables
String url = this.url.toString();
@@ -340,6 +340,9 @@
//Update tile coordinate
req.tile = new mapproxy.tms.TileServiceGrid(grid).internal_tile_coord(req.tile, true);
+ if (req.tile == null) {
+ throw new InvalidTileException();
+ }
//Get Image
@@ -355,17 +358,20 @@
Cache cache = new Cache(cacheManager, grid, true);
- try{
+ try {
- //Find the requested tile in the cache. Note that this will throw an error if the tile doesn't exist
- javaxt.io.File file = new javaxt.io.File(cache.tile(req.tile).location);
- if (file.exists() && file.getSize()>0){
- if (file.getExtension().equalsIgnoreCase(format)){
+ // Find the requested tile in the cache. Note that this will
+ // throw an error if the tile doesn't exist
+ javaxt.io.File file = new javaxt.io.File(
+ cache.tile(req.tile).location);
+ if (file.exists() && file.getSize() > 0) {
+ if (file.getExtension().equalsIgnoreCase(format)) {
rsp = file.getBytes().toByteArray();
- }
- else{
+ } else {
rsp = file.getImage().getByteArray(format);
}
+ } else {
+ throw new Exception("No Tile found");
}
}
catch(Exception e){
Modified: core/Status.java
===================================================================
--- core/Status.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/Status.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -10,10 +10,16 @@
public class Status {
- private int numTiles = 0;
+ private int numTilesSuccess = 0;
+ private int numTilesFail = 0;
private int totalTiles = -1;
private long lastUpdate = -1;
-
+ private String lastStatus = "";
+ public static String RUNNING = "Running";
+ public static String COMPLETED = "Completed";
+ public static String STOPPED = "Stopped";
+ public static String FAILED = "Failed";
+
//**************************************************************************
//** Constructor
//**************************************************************************
@@ -34,7 +40,7 @@
public double getPercentComplete(){
if (totalTiles>0){
- return (double)numTiles/(double)totalTiles;
+ return (double)(numTilesSuccess+numTilesFail)/(double)totalTiles;
}
else{
return 0;
@@ -42,20 +48,34 @@
}
public boolean isComplete(){
- return (numTiles==totalTiles);
+ return (numTilesSuccess==totalTiles);
}
- protected void updateTileCount(){
- numTiles++;
+ protected void updateTileCount(boolean success){
+ if (success) {
+ numTilesSuccess++;
+ }
+ else {
+ numTilesFail++;
+ }
lastUpdate = new java.util.Date().getTime();
}
public int getNumTilesProcessed(){
- return this.numTiles;
+ return this.numTilesSuccess;
}
+ public int getNumTilesFailed(){
+ return this.numTilesFail;
+ }
public int getTotalTiles(){
return this.totalTiles;
}
-
+ public String getLastStatus(){
+ return this.lastStatus;
+ }
+ public void setLastStatus(String value){
+ this.lastStatus = value;
+ }
+
}
Added: core/TileConsumer.java
===================================================================
--- core/TileConsumer.java (rev 0)
+++ core/TileConsumer.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -0,0 +1,9 @@
+package mapproxy.core;
+
+public interface TileConsumer
+{
+ void init(int numThreads);
+ void noMoreTiles();
+ void shutdown();
+ void processTile(TileListener callback, Layer layer, int[] tile, boolean invalidate);
+}
\ No newline at end of file
Added: core/TileListener.java
===================================================================
--- core/TileListener.java (rev 0)
+++ core/TileListener.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -0,0 +1,6 @@
+package mapproxy.core;
+
+public interface TileListener
+{
+ void doneProcessingTile(boolean success);
+}
\ No newline at end of file
Modified: core/TileSeeder.java
===================================================================
--- core/TileSeeder.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/TileSeeder.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -1,291 +1,375 @@
package mapproxy.core;
import mapproxy.core.cache.*;
import mapproxy.core.grid.TileGrid;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
//******************************************************************************
//** TileSeeder Class
//******************************************************************************
/**
- * Thread used to generate a tile cache
- *
+ * Thread used to generate a tile cache
+ *
******************************************************************************/
-public class TileSeeder implements Runnable {
+public class TileSeeder implements TileListener {
+ ExecutorService _exec = null;
+ private static int MAX_TILES_IN_PROCESS = 25;
+ private Status status = null;
+ private Integer tilesInQueue = 0;
+ private TileIterator iterator = null;
+ private TileConsumer tileProcessor = null;
+ boolean isStopped=false;
- private static java.util.List<Object[]> seeds = new java.util.LinkedList<Object[]>();
- private static ConcurrentHashMap<Integer, Layer> layers = new ConcurrentHashMap<Integer, Layer>();
+ //**************************************************************************
+ //** Constructor
+ //**************************************************************************
+ /** Creates a new instance of TileSeeder. */
+ public TileSeeder() {
+ }
- private static ConcurrentHashMap<Integer, Status> stats = new ConcurrentHashMap<Integer, Status>();
-
-
//**************************************************************************
- //** TileIterator
+ //** TileIterator Class
//**************************************************************************
- /** Class used to iterate through tiles and add them to the queue */
+ /** Class used to iterate through tiles and add them to the queue */
- private static class TileIterator implements Runnable {
- int layerID;
+ private static class TileIterator extends Thread {
+ private Layer layer = null;
private double[] bbox;
private int startLevel;
private int endLevel;
- private TileIterator(int layerID, double[] bbox, int startLevel, int endLevel){
- this.layerID = layerID;
+ private boolean invalidate;
+ private TileSeeder tileSeeder = null;
+
+ private TileIterator(Layer layer, TileSeeder tileSeeder,
+ double[] bbox, int startLevel, int endLevel, boolean invalidate) {
+ this.layer = layer;
+ this.tileSeeder = tileSeeder;
this.bbox = bbox;
this.startLevel = startLevel;
this.endLevel = endLevel;
+ this.invalidate = invalidate;
}
- public void run(){
+ public void cancel() {
+ //System.err.println("TileIterator interrupted");
+ this.interrupt();
+ }
+ public void run() {
+ try {
+ tileSeeder.updateTotalTiles(-1);
+ // System.err.println("In TileIterator");
+ int totalTiles = 0;
+ TileGrid grid = (TileGrid) layer.getParam("grid");
+ if (Thread.currentThread().isInterrupted()) {
+ tileSeeder.noMoreTiles();
+ return;
+ }
- Layer layer;
- synchronized (layers) {
- layer = layers.get(layerID);
- layers.notifyAll();
- }
+ List<Integer> levels = null;
+ if (startLevel == -1 && endLevel == -1) {
+ levels = layer.getCachedLevels();
+ } else {
+ levels = new ArrayList<Integer>();
+ for (int level = startLevel; level < endLevel + 1; level++) {
+ levels.add(level);
+ }
+ }
+ List<Generator<int[]>> generators = new ArrayList<Generator<int[]>>();
+ for (int l = 0; l < levels.size(); l++) {
+ if (Thread.currentThread().isInterrupted()) {
+ tileSeeder.noMoreTiles();
+ return;
+ }
+ int level = levels.get(l);
+ Object[] arr = _create_tile_iterator(grid, bbox, level);
+ int est_number_of_tiles = (Integer) arr[0];
+ Generator<int[]> tiles = (Generator<int[]>) arr[1];
+ generators.add(tiles);
+ totalTiles += est_number_of_tiles;
+ }
+ tileSeeder.updateTotalTiles(totalTiles);
- TileGrid grid = (TileGrid) layer.getParam("grid");
-
-
- int numTiles = 0;
- for (int level=startLevel; level<endLevel+1; level++){
- Object[] arr = _create_tile_iterator(grid, bbox, level);
- //int est_number_of_tiles = (Integer) arr[0];
- Generator<int[]> tiles = (Generator<int[]>) arr[1];
-
-
- for (int[] tile : tiles){
-
- synchronized (seeds) {
- seeds.add(seeds.size(), new Object[]{layerID, tile});
- seeds.notifyAll();
+ // System.err.println("total tiles " + totalTiles);
+ int addedCnt = 0;
+ for (int l = 0; l < generators.size(); l++) {
+ Generator<int[]> tiles = generators.get(l);
+ for (int[] tile : tiles) {
+ if (Thread.currentThread().isInterrupted()) {
+ tileSeeder.noMoreTiles();
+ return;
+ }
+ boolean added = false;
+ while (tileSeeder.waitToAddTile()) {
+ try {
+ // System.err.println("waiting to add tile");
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // System.err.println("Leaving TileIterator - interrupted");
+ tileSeeder.noMoreTiles();
+ return;
+ }
+ }
+ tileSeeder.processTile(layer, tile, invalidate);
}
-
- numTiles++;
-
}
-
+ // System.err.println("TileIterator DONE total tiles " +
+ // totalTiles);
+ tileSeeder.noMoreTiles();
+ } catch (Exception e) {
+ tileSeeder.noMoreTiles();
}
+ // System.err.println("Leaving TileIterator.");
+ }
+ }
- //Update the stats for this layer
- synchronized (stats){
- Status status = stats.get(layerID);
- synchronized (status){
- status.setTotal(numTiles);
- status.notifyAll();
- }
-
- //stats.get(layerID).setTotal(numTiles);
- stats.notifyAll();
+ public boolean waitToAddTile() {
+ synchronized (tilesInQueue) {
+ if (tilesInQueue >= MAX_TILES_IN_PROCESS) {
+ return true;
}
+ }
+ return false;
+ }
-
- /*
- //Add null tile to notify the threads that we are done
- synchronized (seeds) {
- seeds.add(seeds.size(), new Object[]{layerID, numTiles});
- seeds.notifyAll();
+ public void doneProcessingTile(boolean success) {
+ synchronized (tilesInQueue) {
+ tilesInQueue--;
+ }
+ // System.err.println("before updating tile cnt");
+ synchronized (status) {
+ status.updateTileCount(success);
+ if (status.getNumTilesProcessed() == status.getTotalTiles()) {
+ status.setLastStatus(Status.COMPLETED);
+ } else {
+ if (status.getNumTilesFailed() != 0) {
+ if (status.getNumTilesFailed() + status.getNumTilesProcessed() == status.getTotalTiles()) {
+ status.setLastStatus(Status.FAILED);
+ }
+ else {
+ status.setLastStatus(Status.RUNNING);
+ }
}
- */
+ else {
+ status.setLastStatus(Status.RUNNING);
+ }
+ }
+
+ // System.err.println("after updating tile cnt");
+ status.notifyAll();
+ }
+ }
+ public void noMoreTiles() {
+ tileProcessor.noMoreTiles();
+ }
+ public void processTile(Layer layer, int[] tile, boolean invalidate) {
+ if (isStopped) {
+ return;
}
+ synchronized (tilesInQueue) {
+ tilesInQueue++;
+ }
+ // System.err.println("before calling seedtask");
+ tileProcessor.processTile(this, layer, tile, invalidate);
}
//**************************************************************************
- //** Constructor
+ //** _create_tile_iterator
//**************************************************************************
- /** Creates a new instance of TileSeeder. */
+ /** Return all tiles that intersect the `bbox` on `level`.
+ * @return estimated number of tiles, tile iterator
+ */
+ private static Object[] _create_tile_iterator(TileGrid grid, double[] bbox,
+ int level) {
- public TileSeeder(){
+ double res = grid.resolution(level);
+ int[] pixelSize = bbox_pixel_size(bbox, res);
+ int w = pixelSize[0];
+ int h = pixelSize[1];
+
+ Object[] affected_tiles = grid.get_affected_tiles(bbox, pixelSize,
+ null, false);
+ bbox = (double[]) affected_tiles[0];
+ int[] extents = (int[]) affected_tiles[1];
+ int est_number_of_tiles = extents[0] * extents[1];
+ Generator<int[]> tiles = (Generator<int[]>) affected_tiles[2];
+ return new Object[] { est_number_of_tiles, tiles };
}
-
//**************************************************************************
- //** addLayer
+ //** bbox_pixel_size
//**************************************************************************
- /** Used to add a layer for the TileSeeder to process. */
+ /** Return the size of the `bbox` in pixel at the given `res`.
+ */
+ private static int[] bbox_pixel_size(double[] bbox, double res) {
+ double w = bbox[2] - bbox[0];
+ double h = bbox[3] - bbox[1];
+ return new int[] { cint(w / res), cint(h / res) };
+ }
- public static Status addLayer(Layer layer, double[] bbox, int startLevel, int endLevel){
+ private static int cint(double d) {
+ return javaxt.utils.string.toInt(d);
+ }
+ // **************************************************************************
+ // ** execute
+ // **************************************************************************
+ /** Used to add a layer for the TileSeeder to process. */
+ public Status execute(Layer layer, int numThreads, double[] bbox,
+ int startLevel, int endLevel, boolean invalidate, TileConsumer tileProcessor) {
+
FileCache cache = (FileCache) layer.getParam("cache");
TileGrid grid = (TileGrid) layer.getParam("grid");
- if (cache==null || grid==null) return null;
+ if (cache == null || grid == null)
+ return null;
-
- int layerID;
- synchronized (layers) {
- layerID = layers.size();
- layers.put(layerID, layer);
- layers.notifyAll();
+ status = new Status();
+ synchronized (status) {
+ status.setTotal(0);
+ status.setLastStatus(Status.RUNNING);
+ status.notifyAll();
}
+
+ if (tileProcessor != null) {
+ this.tileProcessor = tileProcessor;
+ }
+ else {
+ this.tileProcessor = new TileProcessor(this);
+ }
+ this.tileProcessor.init(numThreads);
+
+ iterator = new TileIterator(layer, this, bbox, startLevel, endLevel,
+ invalidate);
+ iterator.start();
-
- Status status = new Status();
-
- synchronized (stats){
- stats.put(layerID, status);
- stats.notifyAll();
- }
-
- new Thread(new TileIterator(layerID, bbox, startLevel, endLevel)).run();
-
return status;
+ }
+ private void updateTotalTiles(int totalTiles) {
+ synchronized (status) {
+ status.setTotal(totalTiles);
+ status.notifyAll();
+ }
}
-
- /*
- public static void stop(){
- synchronized (seeds) {
- seeds.add(seeds.size(), null);
- seeds.notifyAll();
+ public boolean stop() {
+ isStopped = true;
+ try {
+ if (iterator != null) {
+ iterator.cancel();
+ }
+ //System.err.println("TileProcessor shutdown");
+ tileProcessor.shutdown();
+ // System.err.println("TileSeeder Shutdown success ? : " +
+ synchronized (status) {
+ status.setLastStatus(Status.STOPPED);
+ status.notifyAll();
+ }
+ } catch (Exception e) {
+ //System.err.println("TILESEEDER STOP EXCEPTION " + e.getMessage());
+ return false;
}
+ return true;
}
- */
//**************************************************************************
- //** run
+ //** TileProcessor Class
//**************************************************************************
- /** Used to process tiles added to the queue. */
+ /** Class used to iterate through tiles and add them to the queue */
- public void run(){
- while (true) {
-
- //Find request in pool
- Object[] arr;
- synchronized (seeds) {
- while (seeds.isEmpty()) {
- try {
- seeds.wait();
- }
- catch (InterruptedException e) {
- e.printStackTrace();
- }
+ public class TileProcessor implements TileConsumer {
+ TileListener callback=null;
+ public TileProcessor (TileListener callback) {
+ this.callback = callback;
+ }
+ public void init(int numThreads) {
+ _exec = Executors.newFixedThreadPool(numThreads);
+ }
+ public void noMoreTiles() {
+ _exec.shutdown();
+ boolean result = false;
+ try {
+ // System.out.println("Awaiting map cache task completion");
+ result = _exec.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
+ } catch (InterruptedException ie) {
+ // TODO: Log here
+ // System.out.println("InterruptedException");
}
- arr = (Object[]) seeds.remove(0);
- if (arr==null){
- seeds.add(null);
- seeds.notifyAll();
- return;
- }
- }
+ // System.out.println("result = " + result);
+ // System.err.println("shutdown complete");
+ }
+ public void shutdown() {
+ _exec.shutdownNow();
+ }
+ public void processTile(TileListener callback, Layer layer, int[] tile, boolean invalidate) {
+ _exec.execute(new SeedTask(layer, callback, tile, invalidate));
+ }
+ }
+ //**************************************************************************
+ //** SeedTask
+ //**************************************************************************
+ /** Class used to iterate through tiles and add them to the queue */
+ public class SeedTask implements Runnable {
+ Layer layer;
+ TileListener tileListener;
+ int[] tile;
+ boolean invalidate;
- Layer layer;
- int layerID = (Integer) arr[0];
- synchronized (layers) {
- layer = layers.get(layerID);
- layers.notifyAll();
- }
+ public SeedTask(Layer layer, TileListener tileListener, int[] tile,
+ boolean invalidate) {
+ this.layer = layer;
+ this.tileListener = tileListener;
+ this.tile = tile;
+ this.invalidate = invalidate;
+ }
- Object obj = arr[1];
- if (obj instanceof int[]){
- int[] tile = (int[]) obj;
- FileCache cache = (FileCache) layer.getParam("cache");
- TileGrid grid = (TileGrid) layer.getParam("grid");
- try {
- layer.getImage(grid.tile_size[0], grid.tile_size[1], cache.file_ext, grid.tile_bbox(tile[0],tile[1],tile[2]), grid.srs.toString());
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- /*
- System.out.println(Python.cstr(tile));
- try{
- Thread.sleep(500);
- }
- catch(Exception e){}
- */
- synchronized (stats){
- //Integer numTiles = (Integer) stats.get(layerID).get("Total Processed");
- //if (numTiles==null) numTiles = 0;
- Status status = stats.get(layerID);
- synchronized (status){
- status.updateTileCount();
- status.notifyAll();
- }
-
- stats.notifyAll();
- }
-
-
- /*
- synchronized (stats){
- Integer total = (Integer) stats.get(layerID).get("Total Files");
- if (total!=null){
- Integer numTiles = (Integer) stats.get(layerID).get("Total Processed");
- if (numTiles==null) numTiles = 0;
- if (numTiles==total){
- stats.get(layerID).put("Is Complete", true);
+ /** Used to process tiles */
+ public void run() {
+ boolean success = true;
+ //System.err.println("in seed cache task");
+ if (tile != null) {
+ FileCache cache = (FileCache) layer.getParam("cache");
+ TileGrid grid = (TileGrid) layer.getParam("grid");
+ try {
+ if (!invalidate) {
+ //System.err.println("in layer image");
+ javaxt.io.Image img = layer.getImage(grid.tile_size[0],
+ grid.tile_size[1], cache.file_ext, grid
+ .tile_bbox(tile[0], tile[1], tile[2]),
+ grid.srs.toString());
+ if (img == null) {
+ success = false;
}
+ } else {
+ // System.err.println("before tileseeder invalidate bbox");
+ layer.invalidateTile(tile, cache.file_ext, grid.srs
+ .toString());
}
- stats.notifyAll();
+ } catch (Exception e) {
+ success = false;
+ e.printStackTrace();
}
- */
-
-
- }
-
-
-
-
- } // end while
+ // System.err.println("done processing tile " + tileid);
+ }
+ tileListener.doneProcessingTile(success);
+ }
}
-
-
-
-
-
- //**************************************************************************
- //** _create_tile_iterator
- //**************************************************************************
- /** Return all tiles that intersect the `bbox` on `level`.
- * @return estimated number of tiles, tile iterator
- */
- private static Object[] _create_tile_iterator(TileGrid grid, double[] bbox, int level){
-
- double res = grid.resolution(level);
- int[] pixelSize = bbox_pixel_size(bbox, res);
- int w = pixelSize[0];
- int h = pixelSize[1];
-
- Object[] affected_tiles = grid.get_affected_tiles(bbox, pixelSize, null, false);
- bbox = (double[]) affected_tiles[0];
- //int[] _grid = (int[]) affected_tiles[1];
- Generator<int[]> tiles = (Generator<int[]>) affected_tiles[2];
-
- int est_number_of_tiles = cint((w/grid.tile_size[0] *
- h/grid.tile_size[1]));
- return new Object[]{est_number_of_tiles, tiles};
- }
-
-
- //**************************************************************************
- //** bbox_pixel_size
- //**************************************************************************
- /** Return the size of the `bbox` in pixel at the given `res`.
- */
- private static int[] bbox_pixel_size(double[] bbox, double res){
- double w = bbox[2] - bbox[0];
- double h = bbox[3] - bbox[1];
- return new int[]{cint(w/res), cint(h/res)};
- }
-
- private static int cint(double d){
- return javaxt.utils.string.toInt(d);
- }
-
-}
+}
\ No newline at end of file
Modified: core/cache/Cache.java
===================================================================
--- core/cache/Cache.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/cache/Cache.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -83,7 +83,7 @@
* @param out_size the target output size
* @return 'ImageSource`
*/
- private TiledImage _tiled_image(double[] req_bbox, SRS req_srs, int[] out_size){
+ private TiledImage _tiled_image(double[] req_bbox, SRS req_srs, int[] out_size) throws Exception {
Object[] arr = this.grid.get_affected_tiles(req_bbox, out_size, req_srs, false);
double[] src_bbox = (double[]) arr[0];
@@ -102,6 +102,9 @@
TileCollection tiles = this._tiles(affected_tile_coords);
javaxt.io.Image[] tile_sources = new javaxt.io.Image[tiles.size()];
for (int i=0; i<tiles.size(); i++){
+ if (tiles.get(i).is_missing()) {
+ throw new Exception("Tile is missing.");
+ }
tile_sources[i] = tiles.get(i).source;
}
@@ -122,7 +125,7 @@
* @param out_size the output size
* @return `ImageSource`
*/
- public javaxt.io.Image image(double[] req_bbox, SRS req_srs, int[] out_size){
+ public javaxt.io.Image image(double[] req_bbox, SRS req_srs, int[] out_size) throws Exception{
if (!this.grid.srs.equals(req_srs)){
System.out.println("src_srs: " + this.grid.srs);
System.out.println("req_srs: " + req_srs);
@@ -130,7 +133,17 @@
TiledImage tiled_image = this._tiled_image(req_bbox, req_srs, out_size);
return tiled_image.transform(req_bbox, req_srs, out_size);
}
+
+ public TileCollection getInvalidTiles(double[] req_bbox, SRS req_srs, int[] out_size) {
+ System.err.println("in cache invalidate bbox");
+ Object[] arr = this.grid.get_affected_tiles(req_bbox, out_size, req_srs, false);
+ double[] src_bbox = (double[]) arr[0];
+ int[] tile_grid = (int[]) arr[1];
+ Generator<int[]> affected_tile_coords = (Generator<int[]>) arr[2];
+ TileCollection tiles = new TileCollection(affected_tile_coords);
+ return tiles;
+ }
//**************************************************************************
Modified: core/cache/_Tile.java
===================================================================
--- core/cache/_Tile.java 2012-01-10 11:42:33 UTC (rev 38)
+++ core/cache/_Tile.java 2012-01-10 11:43:40 UTC (rev 39)
@@ -102,6 +102,9 @@
if (other==null) return false;
if (other instanceof _Tile){
_Tile tile = (_Tile) other;
+ if (tile.coord == null || this.coord == null || tile.source == null || this.source == null) {
+ return false;
+ }
if ( tile.coord.equals(this.coord) && tile.source.equals(this.source)){
return true;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|