From: <bsc...@us...> - 2016-01-18 12:01:34
|
Revision: 20047 http://sourceforge.net/p/unicore/svn/20047 Author: bschuller Date: 2016-01-18 12:01:31 +0000 (Mon, 18 Jan 2016) Log Message: ----------- fix: data triggering on shared storages (#76); fix: REST API does not show mountPoint on Home (#75); some cleanup; add logging categories Modified Paths: -------------- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/SMSProperties.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/UAS.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/cdmi/CDMIStorageAdapterFactory.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/FileTransferImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/ServerToServerFileTransferImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/RandomByteIOImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/StreamableByteIOImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/FileServlet.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/HttpFileTransferImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/UResource.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/rft/StoreImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/enumeration/EnumerationImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/job/JobManagementHomeImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/job/JobManagementImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/reservation/ReservationManagementImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/ConsolidateStorageFactoryInstance.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/FixedStorageImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/HomeStorageImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/InitDefaultStorageFactory.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/InitSharedStorages.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/PathedStorageImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/SMSBaseImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/SMSUtils.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/StorageDescriptionRP.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/StorageFactoryHomeImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/StorageFactoryImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/sms/StorageManagementHomeImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/TSSFrontend.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/TargetSystemFactoryImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/TargetSystemImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/rp/ApplicationsResourceProperty.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/rp/TSSReferenceResourceProperty.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/util/GenerateJMSInstances.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/util/RecreateJMSReferenceList.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/util/RecreateReservationReferenceList.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/impl/tss/util/RecreateXNJSJobs.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/metadata/BaseMetadataManagementImpl.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/metadata/MetadataSupport.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/impl/BatchJobAction.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/impl/ExtractMetadataAction.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/impl/LocalAction.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/impl/SetupDirectoryScan.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/impl/TriggerRunner.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/xnjs/SharedTriggerProcessor.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/trigger/xnjs/TriggerProcessor.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/util/LogUtil.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/xnjs/StorageAdapterFactory.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/xnjs/TSIStorageAdapterFactory.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/xnjs/U6FileTransferBase.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/xnjs/U6FileTransferCreator.java unicorex/trunk/uas-hadoop/src/main/java/de/fzj/unicore/uas/hadoop/HadoopStorageAdapterFactory.java unicorex/trunk/uas-hadoop/src/main/java/de/fzj/unicore/uas/hadoop/SMSHadoopImpl.java unicorex/trunk/uas-hadoop/src/main/java/de/fzj/unicore/uas/hadoop/tsi/HadoopExecution.java unicorex/trunk/uas-hadoop/src/main/java/de/fzj/unicore/uas/hadoop/tsi/YarnUtils.java unicorex/trunk/uas-jclouds/src/main/java/de/fzj/unicore/uas/jclouds/s3/S3StorageAdapter.java unicorex/trunk/uas-jclouds/src/main/java/de/fzj/unicore/uas/jclouds/s3/S3StorageAdapterFactory.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/LuceneIndexer.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/LuceneMetadataManager.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/MetadataCrawler.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/MetadataFile.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/MetadataProperties.java unicorex/trunk/uas-metadata/src/main/java/eu/unicore/uas/metadata/utils/JSONAdapter.java unicorex/trunk/uas-metadata/src/test/java/eu/unicore/uas/metadata/LuceneIndexerTest.java unicorex/trunk/uas-uftp/src/main/java/de/fzj/unicore/uas/fts/uftp/UFTPConnector.java unicorex/trunk/uas-uftp/src/main/java/de/fzj/unicore/uas/fts/uftp/UFTPCreator.java unicorex/trunk/uas-uftp/src/main/java/de/fzj/unicore/uas/fts/uftp/UFTPFileTransferImpl.java unicorex/trunk/uas-uftp/src/main/java/de/fzj/unicore/uas/fts/uftp/UFTPProperties.java unicorex/trunk/uas-uftp/src/main/java/de/fzj/unicore/uas/fts/uftp/UFTPStartupTask.java unicorex/trunk/uas-xtreemfs/src/main/java/de/fzj/unicore/uas/xtreemfs/XtreemFSDownload.java unicorex/trunk/uas-xtreemfs/src/main/java/de/fzj/unicore/uas/xtreemfs/XtreemFSUpload.java unicorex/trunk/uas-xtreemfs/src/main/java/de/fzj/unicore/uas/xtreemfs/XtreemProperties.java Removed Paths: ------------- unicorex/trunk/uas-core/TODO.txt unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/RandomByteIOSupport.java unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/StorageAdapterRByteIOSupport.java Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/SMSProperties.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/SMSProperties.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/SMSProperties.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -14,8 +14,8 @@ import de.fzj.unicore.uas.impl.sms.DefaultStorageInfoProvider; import de.fzj.unicore.uas.impl.sms.StorageInfoProvider; import de.fzj.unicore.uas.impl.sms.StorageManagementHomeImpl.StorageTypes; +import de.fzj.unicore.uas.util.LogUtil; import de.fzj.unicore.xnjs.io.IStorageAdapter; -import eu.unicore.util.Log; import eu.unicore.util.configuration.ConfigurationException; import eu.unicore.util.configuration.DocumentationReferenceMeta; import eu.unicore.util.configuration.PropertiesHelper; @@ -32,7 +32,7 @@ */ public class SMSProperties extends PropertiesHelper { - private static final Logger log = Log.getLogger(Log.SERVICES, SMSProperties.class); + private static final Logger log = LogUtil.getLogger(LogUtil.CONFIG, SMSProperties.class); // Subkeys of general-purpose configuration properties of storages and storage factories. public static final String NAME="name"; Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/UAS.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/UAS.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/UAS.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -100,7 +100,7 @@ Kernel.addMBean(jmxBean, "UNICORE/X"); } - + @Override public String getConnectionStatus(){ if (!kernel.getContainerProperties().getBooleanValue(ContainerProperties.ON_STARTUP_SELFTEST)) return ""; Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/cdmi/CDMIStorageAdapterFactory.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/cdmi/CDMIStorageAdapterFactory.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/cdmi/CDMIStorageAdapterFactory.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -19,8 +19,6 @@ */ public class CDMIStorageAdapterFactory implements StorageAdapterFactory { - private static final long serialVersionUID = 1L; - @Override public CDMIStorageAdapter createStorageAdapter(UASWSResourceImpl parent) throws IOException { Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/FileTransferImpl.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/FileTransferImpl.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/FileTransferImpl.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -73,7 +73,7 @@ */ public abstract class FileTransferImpl extends UASWSResourceImpl implements DataResource, FileTransfer { - private static final Logger logger=LogUtil.getLogger(LogUtil.SERVICES,FileTransferImpl.class); + private static final Logger logger = LogUtil.getLogger(LogUtil.DATA,FileTransferImpl.class); /** * Init parameter: id of the XNJS action this Filetransfer belongs to Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/ServerToServerFileTransferImpl.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/ServerToServerFileTransferImpl.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/ServerToServerFileTransferImpl.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -41,7 +41,7 @@ */ public class ServerToServerFileTransferImpl extends FileTransferImpl { - private static final Logger logger=LogUtil.getLogger(LogUtil.SERVICES,ServerToServerFileTransferImpl.class); + private static final Logger logger = LogUtil.getLogger(LogUtil.DATA,ServerToServerFileTransferImpl.class); private transient IFileTransfer ft; Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/RandomByteIOImpl.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/RandomByteIOImpl.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/RandomByteIOImpl.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -41,6 +41,7 @@ import javax.xml.namespace.QName; +import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import org.ggf.schemas.byteio.x2005.x10.byteIo.TransferInformationType; import org.ggf.schemas.byteio.x2005.x10.randomAccess.AppendDocument; @@ -71,7 +72,7 @@ */ public class RandomByteIOImpl extends FileTransferImpl implements RandomByteIO { - private static final Logger logger=LogUtil.getLogger(LogUtil.SERVICES,RandomByteIOImpl.class); + private static final Logger logger = LogUtil.getLogger(LogUtil.DATA,RandomByteIOImpl.class); public RandomByteIOImpl(){ super(); @@ -227,7 +228,7 @@ os.write(data,0,read); } if(stride>0){ - if(is!=null)is.close(); + IOUtils.closeQuietly(is); is=createNewInputStream(); long t=offset+stride*(block+1); skipped=is.skip(t); @@ -306,16 +307,12 @@ if(tsi instanceof LocalTS){ boolean export = model.getIsExport(); String file=export?model.getSource():model.getTarget(); - - RandomAccessFile f=((LocalTS)tsi).getRandomAccessFile(file); - try{ + try(RandomAccessFile f=((LocalTS)tsi).getRandomAccessFile(file)){ f.setLength(offset); f.seek(offset); f.write(data); long transferredBytes = model.getTransferredBytes(); model.setTransferredBytes(transferredBytes+data.length); - }finally{ - if(f!=null)f.close(); } } else{ Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/StreamableByteIOImpl.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/StreamableByteIOImpl.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/byteio/StreamableByteIOImpl.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -75,7 +75,7 @@ @MTOM public class StreamableByteIOImpl extends FileTransferImpl implements StreamableByteIO { - private static final Logger logger=LogUtil.getLogger(LogUtil.SERVICES,StreamableByteIOImpl.class); + private static final Logger logger = LogUtil.getLogger(LogUtil.DATA,StreamableByteIOImpl.class); private static final Map<String,Object> streams=Collections.synchronizedMap(new HashMap<String,Object>()); Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/FileServlet.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/FileServlet.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/FileServlet.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -12,11 +12,10 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.log4j.Logger; +import org.apache.commons.io.IOUtils; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.ServletHolder; -import de.fzj.unicore.uas.util.LogUtil; import de.fzj.unicore.wsrflite.Kernel; import de.fzj.unicore.xnjs.io.IFileTransfer.OverwritePolicy; import de.fzj.unicore.xnjs.io.IStorageAdapter; @@ -31,8 +30,6 @@ private static final long serialVersionUID = 1L; - private static final Logger logger=LogUtil.getLogger(LogUtil.SERVICES,FileServlet.class); - private final Map<String,UResource> map=new ConcurrentHashMap<String,UResource>(); private final Map<String,Long> transferredBytes=new ConcurrentHashMap<String,Long>(); @@ -116,21 +113,18 @@ return fr; } - - private static final int BUFFER_SIZE = 2*8192; - @Override protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { UResource r=getResource(request.getRequestURI()); OutputStream out=null; InputStream in =null; try{ - out= r.getOutputStream(); - in= request.getInputStream(); - copy(in,out); + out = r.getOutputStream(); + in = request.getInputStream(); + IOUtils.copy(in,out); response.setStatus(HttpURLConnection.HTTP_NO_CONTENT); }finally{ - if(out!=null)out.close(); + IOUtils.closeQuietly(out); } } @@ -149,33 +143,13 @@ } //OK just read the body try{ - out= r.getOutputStream(); - in= request.getInputStream(); - copy(in,out); + out = r.getOutputStream(); + in = request.getInputStream(); + IOUtils.copy(in,out); response.setStatus(HttpURLConnection.HTTP_NO_CONTENT); }finally{ - if(out!=null)out.close(); + IOUtils.closeQuietly(out); } } - //copy all data from an input stream to an output stream - private void copy(InputStream in, OutputStream out)throws IOException{ - byte[] buffer = new byte[BUFFER_SIZE]; - int len=0; - long total=0; - while (true) - { - len=in.read(buffer,0,BUFFER_SIZE); - if (len<0) - break; - if(len>0){ - out.write(buffer,0,len); - total+=len; - } - } - logger.debug("Total bytes copied : "+total); - out.flush(); - } - - } Modified: unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/HttpFileTransferImpl.java =================================================================== --- unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/HttpFileTransferImpl.java 2016-01-13 09:38:48 UTC (rev 20046) +++ unicorex/trunk/uas-core/src/main/java/de/fzj/unicore/uas/fts/http/HttpFileTransferImpl.java 2016-01-18 12:01:31 UTC (rev 20047) @@ -30,7 +30,7 @@ */ public class HttpFileTransferImpl extends FileTransferImpl{ @@ Diff output truncated at 10000 characters. @@ This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |